diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..a1b1bc8c92fee1cbc0cb1bdfd93f92f462706458 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,44 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/3_eval_3_3d085060cf5254eaf2ca.mp4 filter=lfs diff=lfs merge=lfs -text +iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/2_eval_2_adc4ed4f260e51275bb5.mp4 filter=lfs diff=lfs merge=lfs -text +iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/4_eval_4_faf38f4497652bfc9799.mp4 filter=lfs diff=lfs merge=lfs -text +iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/1_eval_1_9e1d3d257c24ab6b4734.mp4 filter=lfs diff=lfs merge=lfs -text +iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/0_eval_0_8e9e62ad5c72d29c1b12.mp4 filter=lfs diff=lfs merge=lfs -text +iz44h5u3/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +iz44h5u3/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +iz44h5u3/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +iz44h5u3/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +iz44h5u3/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/4_eval_4_77f21e81cd3b0f937629.mp4 filter=lfs diff=lfs merge=lfs -text +lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/2_eval_2_7105aabd534b6cd8cb8e.mp4 filter=lfs diff=lfs merge=lfs -text +lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/3_eval_3_f0346db17ff10377ae08.mp4 filter=lfs diff=lfs merge=lfs -text +lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/0_eval_0_880040d892346c7fbdeb.mp4 filter=lfs diff=lfs merge=lfs -text +lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/1_eval_1_6d027f920cfefcbf4542.mp4 filter=lfs diff=lfs merge=lfs -text +lydwde6h/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +lydwde6h/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +lydwde6h/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +lydwde6h/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +lydwde6h/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/3_eval_3_0390825e5beb34880281.mp4 filter=lfs diff=lfs merge=lfs -text +7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/1_eval_1_c7f634ec4b728ff21a6a.mp4 filter=lfs diff=lfs merge=lfs -text +7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/2_eval_2_e83738fb6ffd08c2daf3.mp4 filter=lfs diff=lfs merge=lfs -text +7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/4_eval_4_c88ccf3ee860d0a1e490.mp4 filter=lfs diff=lfs merge=lfs -text +7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/0_eval_0_ba360af7c8232a7b8ce8.mp4 filter=lfs diff=lfs merge=lfs -text +7j9cp4xk/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7j9cp4xk/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7j9cp4xk/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7j9cp4xk/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7j9cp4xk/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +z0y7vp39/wandb/run-20241231_212731-z0y7vp39/run-z0y7vp39.wandb filter=lfs diff=lfs merge=lfs -text +z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/1_eval_1_eb820ba01fc256486a6c.mp4 filter=lfs diff=lfs merge=lfs -text +z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/4_eval_4_4b071c579be3fd81d6bd.mp4 filter=lfs diff=lfs merge=lfs -text +z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/2_eval_2_8637db17a31d8f9e9a53.mp4 filter=lfs diff=lfs merge=lfs -text +z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/0_eval_0_73394dae2cb5c0e3e76d.mp4 filter=lfs diff=lfs merge=lfs -text +z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/3_eval_3_4f1bea0839448d87829b.mp4 filter=lfs diff=lfs merge=lfs -text +z0y7vp39/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +z0y7vp39/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +z0y7vp39/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +z0y7vp39/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +z0y7vp39/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/7j9cp4xk/.hydra/config.yaml b/7j9cp4xk/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05485f98b3c08bece875a460fe25a0cce7504f90 --- /dev/null +++ b/7j9cp4xk/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1006_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/7j9cp4xk/.hydra/hydra.yaml b/7j9cp4xk/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2682fa5aff18d3da62e30a043ad4b2d21ccdf5b --- /dev/null +++ b/7j9cp4xk/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/7j9cp4xk/.hydra/overrides.yaml b/7j9cp4xk/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/7j9cp4xk/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/7j9cp4xk/episode_rosbags/aligned_depth_to_color_K.npy b/7j9cp4xk/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/7j9cp4xk/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/7j9cp4xk/episode_rosbags/cam_tf_world.npy b/7j9cp4xk/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/7j9cp4xk/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/7j9cp4xk/episode_rosbags/color_K.npy b/7j9cp4xk/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/7j9cp4xk/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/7j9cp4xk/episode_rosbags/depth_K.npy b/7j9cp4xk/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/7j9cp4xk/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/7j9cp4xk/eval_robot.log b/7j9cp4xk/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..8b729ca4505cb95315e628feb18e3b61a95e6940 --- /dev/null +++ b/7j9cp4xk/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-01 00:06:05,328][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 00:06:05,331][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/7j9cp4xk/eval_video/0_eval.mp4 b/7j9cp4xk/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3110306b7f2a627b77c4293813a5f8e3e070c5f1 --- /dev/null +++ b/7j9cp4xk/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba360af7c8232a7b8ce8aaa030dd270211880cfd5b254c4bc04bf8a43662e7a2 +size 1282700 diff --git a/7j9cp4xk/eval_video/1_eval.mp4 b/7j9cp4xk/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b0c8d2e541acefd713fbd3bdfbaf8a1a6c820a8a --- /dev/null +++ b/7j9cp4xk/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7f634ec4b728ff21a6acbff209564236673efe05774bd33a9431331b430c7a1 +size 1434311 diff --git a/7j9cp4xk/eval_video/2_eval.mp4 b/7j9cp4xk/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5e1aa4c7a4a88582b00a6d24b0e828312b429d70 --- /dev/null +++ b/7j9cp4xk/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e83738fb6ffd08c2daf38b899a4b987a92c44d56b7a7e79502a841a7273f8c95 +size 870326 diff --git a/7j9cp4xk/eval_video/3_eval.mp4 b/7j9cp4xk/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..532782ddef9c2c80f622e032e211f140c2abf63b --- /dev/null +++ b/7j9cp4xk/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0390825e5beb348802816ef3517f3a8a887d453cb1d4be00750cfb7558d508ad +size 907684 diff --git a/7j9cp4xk/eval_video/4_eval.mp4 b/7j9cp4xk/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ffb9f335a4fb91eff8ec8561450e2e6e3de97cb9 --- /dev/null +++ b/7j9cp4xk/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c88ccf3ee860d0a1e490c8672df176570ddde0118f4d935801d393e7925c4286 +size 1540490 diff --git a/7j9cp4xk/tb/events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 b/7j9cp4xk/tb/events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 new file mode 100644 index 0000000000000000000000000000000000000000..40010f90c3d66d4760317e728003023e1dd5c485 --- /dev/null +++ b/7j9cp4xk/tb/events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:790d779d6a4b2ea5d23d7f3f05599e89fb88c604e0196694f5ad3c6c43f7835f +size 6519 diff --git a/7j9cp4xk/wandb/debug-internal.log b/7j9cp4xk/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f761ac903b31c879afd967b082d5778e40db6846 --- /dev/null +++ b/7j9cp4xk/wandb/debug-internal.log @@ -0,0 +1,1031 @@ +2025-01-01 00:06:11,317 INFO StreamThr :1773737 [internal.py:wandb_internal():85] W&B internal server running at pid: 1773737, started at: 2025-01-01 00:06:11.317096 +2025-01-01 00:06:11,318 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status +2025-01-01 00:06:11,329 INFO WriterThread:1773737 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/run-7j9cp4xk.wandb +2025-01-01 00:06:11,330 DEBUG SenderThread:1773737 [sender.py:send():379] send: header +2025-01-01 00:06:11,338 DEBUG SenderThread:1773737 [sender.py:send():379] send: run +2025-01-01 00:06:11,569 INFO SenderThread:1773737 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files +2025-01-01 00:06:11,569 INFO SenderThread:1773737 [sender.py:_start_run_threads():1188] run started: 7j9cp4xk with start time 1735707971.317291 +2025-01-01 00:06:11,593 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 00:06:11,593 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: check_version +2025-01-01 00:06:11,681 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 00:06:11,686 DEBUG HandlerThread:1773737 [system_info.py:__init__():26] System info init +2025-01-01 00:06:11,686 DEBUG HandlerThread:1773737 [system_info.py:__init__():41] System info init done +2025-01-01 00:06:11,686 INFO HandlerThread:1773737 [system_monitor.py:start():194] Starting system monitor +2025-01-01 00:06:11,686 INFO SystemMonitor:1773737 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 00:06:11,686 INFO HandlerThread:1773737 [system_monitor.py:probe():214] Collecting system info +2025-01-01 00:06:11,687 INFO SystemMonitor:1773737 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 00:06:11,688 INFO SystemMonitor:1773737 [interfaces.py:start():188] Started disk monitoring +2025-01-01 00:06:11,688 INFO SystemMonitor:1773737 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 00:06:11,689 INFO SystemMonitor:1773737 [interfaces.py:start():188] Started memory monitoring +2025-01-01 00:06:11,689 INFO SystemMonitor:1773737 [interfaces.py:start():188] Started network monitoring +2025-01-01 00:06:11,699 DEBUG HandlerThread:1773737 [system_info.py:probe():152] Probing system +2025-01-01 00:06:11,700 DEBUG HandlerThread:1773737 [system_info.py:_probe_git():137] Probing git +2025-01-01 00:06:11,706 DEBUG HandlerThread:1773737 [system_info.py:_probe_git():145] Probing git done +2025-01-01 00:06:11,706 DEBUG HandlerThread:1773737 [system_info.py:probe():200] Probing system done +2025-01-01 00:06:11,706 DEBUG HandlerThread:1773737 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T05:06:11.699195', 'startedAt': '2025-01-01T05:06:11.308943', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4123.861625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.902, 'min': 800.0, 'max': 4700.0}, {'current': 4498.097, 'min': 800.0, 'max': 4700.0}, {'current': 4498.146, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.004, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 641.4450988769531}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 00:06:11,706 INFO HandlerThread:1773737 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 00:06:11,706 INFO HandlerThread:1773737 [system_monitor.py:probe():227] Publishing system info +2025-01-01 00:06:11,707 DEBUG HandlerThread:1773737 [system_info.py:_save_code():44] Saving code +2025-01-01 00:06:11,708 DEBUG HandlerThread:1773737 [system_info.py:_save_code():67] Saving code done +2025-01-01 00:06:11,708 DEBUG HandlerThread:1773737 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 00:06:11,771 DEBUG HandlerThread:1773737 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 00:06:11,771 INFO HandlerThread:1773737 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 00:06:11,774 DEBUG SenderThread:1773737 [sender.py:send():379] send: files +2025-01-01 00:06:11,774 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 00:06:11,774 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 00:06:11,775 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 00:06:12,014 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 00:06:12,014 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: python_packages +2025-01-01 00:06:12,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:06:12,016 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:06:12,025 INFO wandb-upload_2:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/gvrny1vq-diff.patch +2025-01-01 00:06:12,040 INFO wandb-upload_0:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/fd0e1vx3-wandb-metadata.json +2025-01-01 00:06:12,053 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/se8x2jqq-code/FISH/eval_robot.py +2025-01-01 00:06:12,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:12,057 INFO HandlerThread:1773737 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/tb" + save: true +} +control { + relay_id: "e089ddc1df5a4e7a9851f3eb0aa21762" +} +_info { + stream_id: "7j9cp4xk" +} + +2025-01-01 00:06:12,057 DEBUG HandlerThread:1773737 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 00:06:12,098 DEBUG SenderThread:1773737 [sender.py:send():379] send: telemetry +2025-01-01 00:06:12,108 DEBUG SenderThread:1773737 [sender.py:send():379] send: tbrecord +2025-01-01 00:06:12,109 DEBUG SenderThread:1773737 [sender.py:send():379] send: files +2025-01-01 00:06:12,109 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 with policy live +2025-01-01 00:06:12,291 INFO wandb-upload_2:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/edog103e-events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:06:12,571 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-metadata.json +2025-01-01 00:06:12,571 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:06:12,571 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:06:12,571 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/diff.patch +2025-01-01 00:06:12,571 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/code/FISH/eval_robot.py +2025-01-01 00:06:12,572 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/requirements.txt +2025-01-01 00:06:12,572 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/code +2025-01-01 00:06:12,572 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/code/FISH +2025-01-01 00:06:13,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:14,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:14,570 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:06:15,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:16,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:17,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:06:17,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:18,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:19,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:20,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:21,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:22,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:23,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:06:23,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:24,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:25,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:26,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:27,014 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:06:27,015 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:06:27,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:28,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:28,159 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:06:29,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:30,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:31,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:32,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:32,133 DEBUG SenderThread:1773737 [sender.py:send():379] send: config +2025-01-01 00:06:33,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:33,178 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:06:34,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:34,575 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:06:35,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:36,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:37,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:38,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:38,185 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:06:38,577 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:06:39,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:40,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:41,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:42,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:06:42,015 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:06:42,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:43,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:43,250 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:06:43,578 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/config.yaml +2025-01-01 00:06:44,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:44,578 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:06:45,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:46,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:47,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:48,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:48,579 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:06:49,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:06:49,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:50,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:51,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:52,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:52,580 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:06:53,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:54,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:54,581 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:06:54,682 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:06:55,020 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:56,020 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:56,582 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:06:57,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:06:57,015 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:06:57,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:58,020 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:59,020 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:00,020 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:00,148 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:01,020 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:02,020 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:03,020 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:04,020 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:05,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:06,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:06,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:07,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:08,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:09,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:10,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:11,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:11,366 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:11,690 DEBUG SystemMonitor:1773737 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 00:07:11,690 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:07:12,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:07:12,015 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:07:12,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:12,586 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:07:13,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:14,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:15,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:16,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:16,377 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:16,587 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:07:17,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:18,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:18,588 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:07:19,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:20,023 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:20,588 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:07:21,023 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:22,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:22,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:23,023 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:24,023 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:25,024 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:26,024 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:27,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:07:27,015 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:07:27,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:27,151 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:28,024 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:29,024 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:30,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:31,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:32,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:33,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:33,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:34,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:35,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:36,026 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:37,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:38,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:38,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:38,593 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:07:39,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:40,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:41,026 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:41,691 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:07:42,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:07:42,016 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:07:42,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:43,027 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:43,082 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:44,028 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:45,029 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:46,028 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:47,028 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:48,028 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:48,516 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:49,029 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:49,504 DEBUG SenderThread:1773737 [sender.py:send():379] send: files +2025-01-01 00:07:49,504 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_ba360af7c8232a7b8ce8.mp4 with policy now +2025-01-01 00:07:49,504 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:07:49,505 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:07:49,505 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:07:49,506 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:07:49,596 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:07:49,596 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/0_eval_0_ba360af7c8232a7b8ce8.mp4 +2025-01-01 00:07:49,596 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos +2025-01-01 00:07:49,596 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval +2025-01-01 00:07:49,596 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media +2025-01-01 00:07:49,753 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/4t98gag9-media/videos/eval/0_eval_0_ba360af7c8232a7b8ce8.mp4 +2025-01-01 00:07:50,004 INFO wandb-upload_2:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/4d6045ve-media/videos/eval/0_eval_0_ba360af7c8232a7b8ce8.mp4 +2025-01-01 00:07:50,029 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:50,597 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:07:51,029 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:52,029 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:53,029 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:54,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:54,030 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:55,030 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:56,030 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:56,599 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:07:57,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:07:57,016 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:07:57,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:58,030 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:59,030 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:59,153 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:00,031 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:00,601 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:01,031 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:02,031 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:02,601 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:03,031 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:04,031 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:04,373 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:04,602 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:05,032 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:06,032 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:07,032 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:08,032 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:09,032 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:10,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:10,032 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:11,032 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:11,691 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:08:12,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:08:12,016 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:08:12,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:12,605 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:08:12,846 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/mkn3t4aj-events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:08:13,033 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:14,033 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:14,606 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:15,033 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:15,172 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:16,033 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:17,033 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:18,034 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:18,607 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:19,034 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:20,034 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:20,608 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:20,791 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:21,036 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:22,036 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:22,608 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:23,036 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:24,036 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:25,036 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:26,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:26,037 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:27,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:08:27,016 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:08:27,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:28,037 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:29,037 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:30,037 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:31,038 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:31,100 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:32,037 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:33,037 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:34,037 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:35,037 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:36,037 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:37,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:37,038 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:38,038 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:39,038 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:40,038 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:41,038 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:41,692 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:08:42,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:08:42,016 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:08:42,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:42,167 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:42,615 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:43,039 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:44,039 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:45,039 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:46,039 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:47,039 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:48,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:48,039 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:49,040 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:50,040 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:51,041 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:52,041 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:53,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:53,041 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:54,041 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:55,041 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:56,041 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:56,620 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:57,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:08:57,021 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:08:57,043 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:57,122 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:08:57,125 DEBUG SenderThread:1773737 [sender.py:send():379] send: files +2025-01-01 00:08:57,125 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_c7f634ec4b728ff21a6a.mp4 with policy now +2025-01-01 00:08:57,125 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:08:57,126 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:08:57,126 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:08:57,401 INFO wandb-upload_2:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/urucpi4t-media/videos/eval/1_eval_1_c7f634ec4b728ff21a6a.mp4 +2025-01-01 00:08:57,620 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:08:57,621 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/1_eval_1_c7f634ec4b728ff21a6a.mp4 +2025-01-01 00:08:57,621 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval +2025-01-01 00:08:57,924 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/6gabbvs4-media/videos/eval/1_eval_1_c7f634ec4b728ff21a6a.mp4 +2025-01-01 00:08:58,048 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:58,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:58,621 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:59,045 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:59,621 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:09:00,050 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:01,048 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:02,049 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:03,046 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:03,197 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:04,050 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:04,623 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:09:05,051 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:06,053 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:07,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:08,053 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:08,209 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:08,624 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:09:09,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:10,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:10,625 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:09:11,051 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:11,692 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:09:12,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:09:12,018 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:09:12,065 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:12,626 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:09:13,051 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:14,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:14,150 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:15,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:16,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:17,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:18,053 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:19,053 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:20,029 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:20,053 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:21,054 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:22,053 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:23,053 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:24,053 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:25,055 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:26,028 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:26,055 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:27,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:09:27,017 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:09:27,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:28,054 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:29,054 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:30,054 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:31,055 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:31,082 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:32,054 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:33,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:34,056 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:35,055 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:36,055 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:37,028 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:37,055 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:38,055 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:39,055 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:40,055 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:41,056 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:41,693 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:09:42,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:09:42,022 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:09:42,065 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:42,170 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:43,056 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:44,056 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:45,058 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:46,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:47,056 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:48,031 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:48,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:49,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:50,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:51,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:52,059 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:53,060 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:54,030 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:54,058 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:55,058 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:56,059 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:57,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:09:57,017 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:09:57,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:58,059 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:59,060 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:59,164 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:00,059 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:01,060 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:02,060 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:03,063 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:04,060 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:05,030 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:05,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:06,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:07,062 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:08,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:09,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:10,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:11,031 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:11,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:11,693 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:10:12,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:10:12,017 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:10:12,065 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:13,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:14,063 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:15,062 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:16,062 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:16,175 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:17,062 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:18,062 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:19,065 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:20,062 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:21,064 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:22,031 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:22,063 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:23,063 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:24,063 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:25,063 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:26,064 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:27,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:10:27,018 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:10:27,063 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:27,079 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:28,063 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:29,066 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:30,064 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:31,065 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:32,066 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:33,032 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:33,065 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:34,066 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:35,066 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:36,066 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:37,068 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:38,033 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:38,066 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:39,066 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:40,066 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:41,067 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:41,694 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:10:42,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:10:42,019 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:10:42,067 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:43,067 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:43,173 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:43,655 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:10:43,909 INFO wandb-upload_2:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/3in8ytud-events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:10:44,067 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:44,646 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:10:44,646 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:10:44,646 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:10:44,656 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:10:44,656 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:10:45,068 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:46,068 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:47,068 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:48,072 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:48,508 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:49,071 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:50,068 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:50,658 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:10:51,069 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:52,069 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:53,069 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:54,069 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:54,504 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:54,659 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:10:55,069 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:56,069 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:57,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:10:57,018 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:10:57,069 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:58,070 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:59,069 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:00,070 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:00,104 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:01,070 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:02,070 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:03,070 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:04,070 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:05,071 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:06,034 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:06,071 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:07,071 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:08,074 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:09,074 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:10,074 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:11,074 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:11,537 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:11,694 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:11:12,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:11:12,025 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:11:12,076 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:12,665 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:11:12,805 DEBUG SenderThread:1773737 [sender.py:send():379] send: files +2025-01-01 00:11:12,806 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_e83738fb6ffd08c2daf3.mp4 with policy now +2025-01-01 00:11:12,806 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:11:12,807 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:11:12,807 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:11:12,809 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:11:13,055 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/2a211tmn-media/videos/eval/2_eval_2_e83738fb6ffd08c2daf3.mp4 +2025-01-01 00:11:13,077 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:13,666 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:11:13,666 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/2_eval_2_e83738fb6ffd08c2daf3.mp4 +2025-01-01 00:11:13,666 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval +2025-01-01 00:11:13,945 INFO wandb-upload_2:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/g5lih3yv-media/videos/eval/2_eval_2_e83738fb6ffd08c2daf3.mp4 +2025-01-01 00:11:14,078 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:14,666 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:11:15,080 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:16,089 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:16,839 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:17,092 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:18,090 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:19,091 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:20,091 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:21,091 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:22,042 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:22,100 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:23,093 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:24,095 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:24,669 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:11:25,097 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:26,098 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:27,023 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:11:27,024 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:11:27,097 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:27,138 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:28,100 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:28,671 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:11:29,110 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:30,105 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:31,113 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:32,110 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:33,040 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:33,110 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:34,111 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:35,113 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:36,114 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:37,111 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:38,041 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:38,117 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:39,117 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:40,113 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:40,496 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:11:40,497 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:11:40,497 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:11:40,674 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:11:40,675 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:11:40,675 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:11:41,121 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:41,694 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:11:42,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:11:42,026 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:11:42,118 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:43,118 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:43,125 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:44,117 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:45,118 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:46,118 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:46,677 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:11:47,118 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:48,121 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:48,657 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:49,119 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:50,121 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:50,678 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:11:51,121 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:52,121 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:52,679 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:11:53,121 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:54,043 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:54,121 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:55,122 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:56,122 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:57,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:11:57,022 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:11:57,122 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:58,122 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:59,123 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:59,132 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:00,123 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:01,124 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:02,124 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:03,124 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:04,124 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:05,043 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:05,124 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:06,124 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:07,124 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:08,124 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:09,125 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:10,137 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:10,245 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:10,301 DEBUG SenderThread:1773737 [sender.py:send():379] send: files +2025-01-01 00:12:10,301 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_0390825e5beb34880281.mp4 with policy now +2025-01-01 00:12:10,303 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:12:10,304 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:12:10,304 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:12:10,322 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:12:10,572 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/ywpogo1l-media/videos/eval/3_eval_3_0390825e5beb34880281.mp4 +2025-01-01 00:12:10,685 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:12:10,685 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:10,685 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/3_eval_3_0390825e5beb34880281.mp4 +2025-01-01 00:12:10,685 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval +2025-01-01 00:12:10,959 INFO wandb-upload_2:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/z7yvmmtp-media/videos/eval/3_eval_3_0390825e5beb34880281.mp4 +2025-01-01 00:12:11,147 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:11,685 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:11,695 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:12:12,033 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:12:12,033 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:12:12,149 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:12,686 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:13,143 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:14,145 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:15,147 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:15,352 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:16,149 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:16,687 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:17,147 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:18,152 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:19,150 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:20,149 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:20,452 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:21,153 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:22,159 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:22,689 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:23,153 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:24,157 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:25,155 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:26,156 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:26,309 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:26,690 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:27,029 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:12:27,029 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:12:27,154 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:28,157 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:29,153 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:30,157 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:31,155 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:32,058 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:32,160 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:33,159 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:34,165 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:35,164 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:36,161 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:37,059 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:37,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:37,573 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:12:37,574 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:12:37,574 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:12:37,694 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:12:37,694 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:12:37,937 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/ijntgok8-events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:12:38,201 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:38,694 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:39,177 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:40,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:40,695 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:41,168 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:41,696 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:12:42,024 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:12:42,024 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:12:42,168 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:42,177 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:42,697 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:43,168 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:44,168 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:45,168 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:46,168 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:46,698 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:47,168 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:48,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:48,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:49,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:50,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:51,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:52,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:52,700 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:53,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:54,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:54,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:55,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:56,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:57,024 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:12:57,024 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:12:57,170 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:58,170 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:59,148 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:59,170 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:00,170 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:01,170 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:02,171 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:03,171 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:04,171 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:05,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:05,172 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:06,171 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:07,172 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:08,172 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:09,172 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:10,172 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:10,707 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:10,905 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:11,174 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:11,696 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:13:12,024 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:13:12,024 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:13:12,172 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:13,172 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:14,172 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:15,173 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:15,936 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:16,187 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:16,709 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:17,157 DEBUG SenderThread:1773737 [sender.py:send():379] send: files +2025-01-01 00:13:17,158 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_c88ccf3ee860d0a1e490.mp4 with policy now +2025-01-01 00:13:17,158 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:13:17,158 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:13:17,159 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:17,171 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:17,178 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:17,419 INFO wandb-upload_2:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/u5m8ksmp-media/videos/eval/4_eval_4_c88ccf3ee860d0a1e490.mp4 +2025-01-01 00:13:17,709 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:13:17,709 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/4_eval_4_c88ccf3ee860d0a1e490.mp4 +2025-01-01 00:13:17,709 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval +2025-01-01 00:13:17,993 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/0zgzs2oo-media/videos/eval/4_eval_4_c88ccf3ee860d0a1e490.mp4 +2025-01-01 00:13:18,178 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:18,709 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:19,185 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:20,188 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:21,183 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:21,234 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:22,184 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:23,194 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:24,191 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:24,711 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:25,196 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:26,193 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:26,237 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:27,030 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:13:27,031 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:13:27,197 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:28,198 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:28,713 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:29,210 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:30,207 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:30,714 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:31,204 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:32,076 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:32,207 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:32,714 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:33,211 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:34,227 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:35,218 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:36,220 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:37,076 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:37,220 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:38,223 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:39,217 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:40,229 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:41,150 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 00:13:41,150 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 00:13:41,234 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:41,367 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:13:41,611 INFO wandb-upload_2:1773737 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpe27epawh +2025-01-01 00:13:41,717 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:13:41,718 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 00:13:41,718 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media +2025-01-01 00:13:41,718 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table/eval +2025-01-01 00:13:41,718 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table +2025-01-01 00:13:41,773 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 00:13:41,783 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:13:41,983 INFO SenderThread:1773737 [sender.py:send_request_log_artifact():1525] logged artifact run-7j9cp4xk-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIyODI1Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE4MTA3Mw==', 'latestArtifact': None}} +2025-01-01 00:13:41,983 DEBUG SenderThread:1773737 [sender.py:send():379] send: files +2025-01-01 00:13:41,983 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-01 00:13:41,985 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:13:41,985 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,985 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,985 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:13:41,986 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,986 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,986 DEBUG SenderThread:1773737 [sender.py:send():379] send: telemetry +2025-01-01 00:13:41,986 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,986 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,986 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,987 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,987 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,987 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,988 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,988 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,988 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,988 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,988 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,989 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,989 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,989 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,989 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,990 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,990 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,990 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,990 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,991 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,991 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,991 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,991 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:13:41,991 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 00:13:42,024 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:13:42,206 INFO wandb-upload_0:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/3h0auid4-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 00:13:42,218 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:42,222 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/v1tu7egy-events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:13:42,386 INFO wandb-upload_2:1773737 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjh910wfh +2025-01-01 00:13:42,466 INFO wandb-upload_0:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/rkwxvp6d-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 00:13:42,718 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:13:42,718 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:42,718 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table/eval/total_success_rate_ci_6_0ee4b68a560cd8157b36.table.json +2025-01-01 00:13:42,718 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table/eval +2025-01-01 00:13:42,799 INFO SenderThread:1773737 [sender.py:send_request_log_artifact():1525] logged artifact run-7j9cp4xk-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIyODI2Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE4MTA3NA==', 'latestArtifact': None}} +2025-01-01 00:13:42,799 DEBUG SenderThread:1773737 [sender.py:send():379] send: files +2025-01-01 00:13:42,799 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_0ee4b68a560cd8157b36.table.json with policy now +2025-01-01 00:13:42,800 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:42,801 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:13:42,801 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:42,801 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:42,801 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:13:42,878 DEBUG SenderThread:1773737 [sender.py:send():379] send: exit +2025-01-01 00:13:42,878 INFO SenderThread:1773737 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 00:13:42,878 INFO SenderThread:1773737 [sender.py:send_exit():588] handling runtime: 450 +2025-01-01 00:13:42,878 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:42,878 INFO SenderThread:1773737 [sender.py:send_exit():594] send defer +2025-01-01 00:13:42,879 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:42,879 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 00:13:42,879 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:42,879 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 00:13:42,879 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 1 +2025-01-01 00:13:42,879 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:42,879 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 00:13:42,879 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:42,879 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 00:13:42,879 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 2 +2025-01-01 00:13:42,879 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:42,879 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 00:13:42,879 INFO HandlerThread:1773737 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 00:13:42,880 DEBUG SystemMonitor:1773737 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 00:13:42,880 DEBUG SystemMonitor:1773737 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 00:13:42,880 INFO HandlerThread:1773737 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 00:13:42,880 INFO HandlerThread:1773737 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 00:13:42,932 INFO HandlerThread:1773737 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 00:13:42,933 INFO HandlerThread:1773737 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 00:13:42,933 INFO HandlerThread:1773737 [interfaces.py:finish():200] Joined network monitor +2025-01-01 00:13:42,933 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:42,933 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 00:13:42,933 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 3 +2025-01-01 00:13:42,933 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:13:42,934 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:42,934 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 00:13:42,934 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:42,934 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 00:13:42,934 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 4 +2025-01-01 00:13:42,934 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:42,934 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 00:13:42,995 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/6y9cw5e3-media/table/eval/total_success_rate_ci_6_0ee4b68a560cd8157b36.table.json +2025-01-01 00:13:43,270 INFO wandb-upload_0:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/gv4395un-media/table/eval/total_success_rate_ci_6_0ee4b68a560cd8157b36.table.json +2025-01-01 00:13:43,718 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:13:44,719 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:48,720 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/config.yaml +2025-01-01 00:13:49,644 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:13:49,644 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:49,644 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 00:13:49,644 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:49,646 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 00:13:49,646 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 5 +2025-01-01 00:13:49,646 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:13:49,646 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:13:49,646 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:49,646 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:49,646 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 00:13:49,647 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:49,647 DEBUG SenderThread:1773737 [sender.py:send():379] send: summary +2025-01-01 00:13:49,651 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:49,651 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:49,651 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 00:13:49,651 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 6 +2025-01-01 00:13:49,651 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:49,652 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 00:13:49,652 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:49,652 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 00:13:49,652 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 7 +2025-01-01 00:13:49,652 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:49,652 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:49,652 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 00:13:49,652 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:49,652 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 00:13:49,721 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:13:50,555 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:13:51,251 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 8 +2025-01-01 00:13:51,251 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:13:51,252 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:51,252 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 00:13:51,252 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:51,252 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 00:13:51,252 INFO SenderThread:1773737 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 00:13:51,254 INFO SenderThread:1773737 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 00:13:51,256 INFO SenderThread:1773737 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 00:13:51,269 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 9 +2025-01-01 00:13:51,269 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:51,270 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 00:13:51,270 DEBUG SenderThread:1773737 [sender.py:send():379] send: artifact +2025-01-01 00:13:51,474 INFO SenderThread:1773737 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjk2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjk2Ng==', 'versionIndex': 403}}} +2025-01-01 00:13:51,474 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:51,474 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 00:13:51,474 INFO SenderThread:1773737 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 00:13:51,555 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:13:51,722 INFO SenderThread:1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:51,722 INFO SenderThread:1773737 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files +2025-01-01 00:13:51,722 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/config.yaml config.yaml +2025-01-01 00:13:51,722 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json wandb-summary.json +2025-01-01 00:13:51,722 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/requirements.txt requirements.txt +2025-01-01 00:13:51,722 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-metadata.json wandb-metadata.json +2025-01-01 00:13:51,723 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:13:51,723 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log output.log +2025-01-01 00:13:51,723 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/diff.patch diff.patch +2025-01-01 00:13:51,729 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 00:13:51,729 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/2_eval_2_e83738fb6ffd08c2daf3.mp4 media/videos/eval/2_eval_2_e83738fb6ffd08c2daf3.mp4 +2025-01-01 00:13:51,729 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/4_eval_4_c88ccf3ee860d0a1e490.mp4 media/videos/eval/4_eval_4_c88ccf3ee860d0a1e490.mp4 +2025-01-01 00:13:51,729 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/3_eval_3_0390825e5beb34880281.mp4 media/videos/eval/3_eval_3_0390825e5beb34880281.mp4 +2025-01-01 00:13:51,729 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/1_eval_1_c7f634ec4b728ff21a6a.mp4 media/videos/eval/1_eval_1_c7f634ec4b728ff21a6a.mp4 +2025-01-01 00:13:51,729 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/0_eval_0_ba360af7c8232a7b8ce8.mp4 media/videos/eval/0_eval_0_ba360af7c8232a7b8ce8.mp4 +2025-01-01 00:13:51,729 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table/eval/total_success_rate_ci_6_0ee4b68a560cd8157b36.table.json media/table/eval/total_success_rate_ci_6_0ee4b68a560cd8157b36.table.json +2025-01-01 00:13:51,729 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 00:13:51,729 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 10 +2025-01-01 00:13:51,730 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:13:51,730 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:51,730 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 00:13:51,730 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:51,730 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 00:13:51,730 INFO SenderThread:1773737 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 00:13:51,911 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:13:51,923 INFO wandb-upload_3:1773737 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/config.yaml +2025-01-01 00:13:51,947 INFO wandb-upload_2:1773737 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/requirements.txt +2025-01-01 00:13:51,968 INFO wandb-upload_0:1773737 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:52,168 INFO Thread-11 (_thread_body):1773737 [sender.py:transition_state():614] send defer: 11 +2025-01-01 00:13:52,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:52,169 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 00:13:52,169 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:52,169 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 00:13:52,169 INFO SenderThread:1773737 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 00:13:52,170 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 12 +2025-01-01 00:13:52,170 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:52,170 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 00:13:52,171 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:52,171 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 00:13:52,171 INFO SenderThread:1773737 [file_stream.py:finish():601] file stream finish called +2025-01-01 00:13:52,373 INFO SenderThread:1773737 [file_stream.py:finish():605] file stream finish is done +2025-01-01 00:13:52,373 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 13 +2025-01-01 00:13:52,373 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:52,373 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 00:13:52,373 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:52,373 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 00:13:52,373 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 14 +2025-01-01 00:13:52,373 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:52,374 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 00:13:52,374 DEBUG SenderThread:1773737 [sender.py:send():379] send: final +2025-01-01 00:13:52,374 DEBUG SenderThread:1773737 [sender.py:send():379] send: footer +2025-01-01 00:13:52,374 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:52,374 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 00:13:52,374 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:13:52,375 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:13:52,375 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 00:13:52,375 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: server_info +2025-01-01 00:13:52,376 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:13:52,377 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 00:13:52,377 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 00:13:52,378 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:52,422 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:13:52,422 INFO MainThread:1773737 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 00:13:52,423 INFO MainThread:1773737 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 00:13:52,423 INFO MainThread:1773737 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 00:13:52,423 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 00:13:52,423 INFO HandlerThread:1773737 [handler.py:finish():882] shutting down handler +2025-01-01 00:13:53,377 INFO WriterThread:1773737 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/run-7j9cp4xk.wandb +2025-01-01 00:13:53,422 INFO SenderThread:1773737 [sender.py:finish():1615] shutting down sender +2025-01-01 00:13:53,422 INFO SenderThread:1773737 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 00:13:53,422 INFO SenderThread:1773737 [file_pusher.py:join():175] waiting for file pusher diff --git a/7j9cp4xk/wandb/debug.log b/7j9cp4xk/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..70fedefef334092b98fdb5017db2fa137c87d337 --- /dev/null +++ b/7j9cp4xk/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_setup.py:_flush():76] Configure stats pid to 1773565 +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/settings +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/logs/debug.log +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/logs/debug-internal.log +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_init.py:init():569] calling init triggers +2025-01-01 00:06:11,312 INFO MainThread:1773565 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk'} +2025-01-01 00:06:11,312 INFO MainThread:1773565 [wandb_init.py:init():619] starting backend +2025-01-01 00:06:11,312 INFO MainThread:1773565 [wandb_init.py:init():623] setting up manager +2025-01-01 00:06:11,316 INFO MainThread:1773565 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 00:06:11,317 INFO MainThread:1773565 [wandb_init.py:init():631] backend started and connected +2025-01-01 00:06:11,329 INFO MainThread:1773565 [wandb_init.py:init():720] updated telemetry +2025-01-01 00:06:11,336 INFO MainThread:1773565 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 00:06:11,593 INFO MainThread:1773565 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 00:06:11,631 INFO MainThread:1773565 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 00:06:11,632 INFO MainThread:1773565 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 00:06:12,015 INFO MainThread:1773565 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 00:06:12,015 INFO MainThread:1773565 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 00:06:12,015 INFO MainThread:1773565 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 00:06:12,016 INFO MainThread:1773565 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 00:06:12,017 INFO MainThread:1773565 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 00:06:12,017 INFO MainThread:1773565 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/tb, True +2025-01-01 00:06:32,133 INFO MainThread:1773565 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-01 00:13:53,424 WARNING MsgRouterThr:1773565 [router.py:message_loop():77] message_loop has been closed diff --git a/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/code/FISH/eval_robot.py b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/config.yaml b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ded5175dcf700eef5e9511657949aea734ecfa58 --- /dev/null +++ b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1006_0' +wandb_notes: + desc: null + value: 1006_0_req_1352_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735707971 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/diff.patch b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..625cf877496560323ca22cd6b0a075ba511a2dc6 --- /dev/null +++ b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/diff.patch @@ -0,0 +1,106 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..01bdd86 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1007_0' ++# wandb_run_id: '1002_0' + ++# all books, 6/20 demos per book ++# RGBD+mask+act history ++# wandb_run_id: '1007_0' ++ ++# RGBD+mask+contact(all ftrs)+act history ++wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..50493fe 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -490,7 +490,8 @@ class Workspace: + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] +- total_num_episodes += filtered_run.summary_metrics['episode'] ++ # total_num_episodes += filtered_run.summary_metrics['episode'] ++ total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes +@@ -513,7 +514,6 @@ class Workspace: + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + +- + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index e3031af..6f97e26 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,11 +330,11 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.01 +- self.move_to_rot_tolerance = 2.0 ++ self.move_to_trans_tolerance = 0.005 ++ self.move_to_rot_tolerance = 1.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) +- self.move_to_target_timeout = 15.0 ++ self.move_to_target_timeout = 5.0 + self.move_to_target_max_rotation = 25.0 + # only can use one of these at a time + # assert use_roll + use_pitch + use_yaw <= 1, "Can only use one of roll, pitch, yaw" diff --git a/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table/eval/total_success_rate_ci_6_0ee4b68a560cd8157b36.table.json b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table/eval/total_success_rate_ci_6_0ee4b68a560cd8157b36.table.json new file mode 100644 index 0000000000000000000000000000000000000000..7367d54ac3567b4cbe6963c4d9241d0c30872293 --- /dev/null +++ b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table/eval/total_success_rate_ci_6_0ee4b68a560cd8157b36.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[15, 30, 0.5, 0.2916622460305133, 0.7063640355651114, 0.33155784014255996, 0.6857495158539253, 0.35735600444547144, 0.642512828083146, 0.37515496092551415, 0.6270253248258695]]} \ No newline at end of file diff --git a/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/0_eval_0_ba360af7c8232a7b8ce8.mp4 b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/0_eval_0_ba360af7c8232a7b8ce8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3110306b7f2a627b77c4293813a5f8e3e070c5f1 --- /dev/null +++ b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/0_eval_0_ba360af7c8232a7b8ce8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba360af7c8232a7b8ce8aaa030dd270211880cfd5b254c4bc04bf8a43662e7a2 +size 1282700 diff --git a/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/1_eval_1_c7f634ec4b728ff21a6a.mp4 b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/1_eval_1_c7f634ec4b728ff21a6a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b0c8d2e541acefd713fbd3bdfbaf8a1a6c820a8a --- /dev/null +++ b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/1_eval_1_c7f634ec4b728ff21a6a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7f634ec4b728ff21a6acbff209564236673efe05774bd33a9431331b430c7a1 +size 1434311 diff --git a/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/2_eval_2_e83738fb6ffd08c2daf3.mp4 b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/2_eval_2_e83738fb6ffd08c2daf3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5e1aa4c7a4a88582b00a6d24b0e828312b429d70 --- /dev/null +++ b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/2_eval_2_e83738fb6ffd08c2daf3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e83738fb6ffd08c2daf38b899a4b987a92c44d56b7a7e79502a841a7273f8c95 +size 870326 diff --git a/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/3_eval_3_0390825e5beb34880281.mp4 b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/3_eval_3_0390825e5beb34880281.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..532782ddef9c2c80f622e032e211f140c2abf63b --- /dev/null +++ b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/3_eval_3_0390825e5beb34880281.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0390825e5beb348802816ef3517f3a8a887d453cb1d4be00750cfb7558d508ad +size 907684 diff --git a/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/4_eval_4_c88ccf3ee860d0a1e490.mp4 b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/4_eval_4_c88ccf3ee860d0a1e490.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ffb9f335a4fb91eff8ec8561450e2e6e3de97cb9 --- /dev/null +++ b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/4_eval_4_c88ccf3ee860d0a1e490.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c88ccf3ee860d0a1e490c8672df176570ddde0118f4d935801d393e7925c4286 +size 1540490 diff --git a/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..978a1d96c2ec44599a799ce5100866b8cbab9fa3 --- /dev/null +++ b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log @@ -0,0 +1,292 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735707992.133642]: resetting environment +[INFO] [1735707992.139952]: cleared current plan +[INFO] [1735707992.141340]: moving to home +Move to target timeout: 5.000706434249878. Trans error: 0.005492541939020157. Rot error: 0.6255666878961403 +Move to target timeout: 5.099776268005371. Trans error: 0.00553597724500355. Rot error: 1.351663053219931 +[INFO] [1735708003.244732]: reached home +[INFO] [1735708003.246069]: reset action history +[INFO] [1735708005.955648]: environment reset +Starting episode 0 +[INFO] [1735708005.960121]: resetting environment +[INFO] [1735708005.964759]: cleared current plan +[INFO] [1735708005.965722]: moving to home +Move to target timeout: 5.0008673667907715. Trans error: 0.0055519623945830075. Rot error: 1.351013137124741 +[INFO] [1735708011.969348]: reached home +[INFO] [1735708011.971034]: reset action history +[INFO] [1735708014.681738]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735708030.324654]: resetting environment +[INFO] [1735708030.331428]: cleared current plan +[INFO] [1735708030.332911]: moving to home +Move to target timeout: 5.000875473022461. Trans error: 0.005565465355532215. Rot error: 1.3506388557960278 +[INFO] [1735708036.336308]: reached home +[INFO] [1735708036.337641]: reset action history +[INFO] [1735708039.046504]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/episode_rosbags/episode_0_2025-01-01-00-07-19.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1773565.log +started roslaunch server http://158.130.50.37:46533/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1773858] +process[print_text-2]: started with pid [1773859] +process[rosbag_record-3]: started with pid [1773883] +started bagging! +For topic gripper_width: timestamp difference is 100516384 for nearest: 1735708056819962805 - target: 1735708056719446421 at idx 229 +preempting episode +[INFO] [1735708068.534323]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735708069.292915]: Stored episode 1. +[INFO] [1735708069.295951]: Saving video... +[INFO] [1735708069.493988]: Video saved! +Starting episode 1 +[INFO] [1735708069.506384]: resetting environment +[INFO] [1735708069.515345]: cleared current plan +[INFO] [1735708069.516032]: moving to home +Move to target timeout: 5.001083612442017. Trans error: 0.007109167519956827. Rot error: 1.0723033994764393 +Move to target timeout: 5.1001622676849365. Trans error: 0.005541084539685159. Rot error: 0.9699113348164797 +[INFO] [1735708080.619758]: reached home +[INFO] [1735708080.620853]: reset action history +[INFO] [1735708083.331341]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735708092.073268]: resetting environment +[INFO] [1735708092.079569]: cleared current plan +[INFO] [1735708092.079757]: moving to home +Move to target timeout: 5.001095533370972. Trans error: 0.005114164879741244. Rot error: 1.0630467298419484 +[INFO] [1735708098.082425]: reached home +[INFO] [1735708098.082831]: reset action history +[INFO] [1735708100.791262]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/episode_rosbags/episode_1_2025-01-01-00-08-20.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1773565.log +started roslaunch server http://158.130.50.37:39067/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1774007] +process[print_text-5]: started with pid [1774008] +process[rosbag_record-6]: started with pid [1774032] +started bagging! +For topic gripper_width: timestamp difference is 118813107 for nearest: 1735708120753277371 - target: 1735708120634464264 at idx 247 +[INFO] [1735708135.635027]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1735708136.429168]: Stored episode 2. +[INFO] [1735708136.429400]: Saving video... +[INFO] [1735708137.098964]: Video saved! +Starting episode 2 +[INFO] [1735708137.127761]: resetting environment +[INFO] [1735708137.149098]: cleared current plan +[INFO] [1735708137.149472]: moving to home +[WARN] [1735708136.972891]: Plan exhausted +[WARN] [1735708137.013767]: Plan exhausted +[WARN] [1735708137.052430]: Plan exhausted +[WARN] [1735708137.092383]: Plan exhausted +[WARN] [1735708137.134821]: Plan exhausted +Move to target timeout: 5.001886367797852. Trans error: 0.007577347103506327. Rot error: 1.2155269143160217 +Move to target timeout: 5.004466772079468. Trans error: 0.005655103647181165. Rot error: 0.9950479222283343 +[INFO] [1735708148.164635]: reached home +[INFO] [1735708148.166071]: reset action history +[INFO] [1735708150.894908]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735708243.495897]: resetting environment +[INFO] [1735708243.506706]: cleared current plan +[INFO] [1735708243.506898]: moving to home +Move to target timeout: 5.0007922649383545. Trans error: 0.004788240826849613. Rot error: 1.031383620243127 +[INFO] [1735708249.509247]: reached home +[INFO] [1735708249.509738]: reset action history +[INFO] [1735708252.227189]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/episode_rosbags/episode_2_2025-01-01-00-10-52.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1773565.log +started roslaunch server http://158.130.50.37:33577/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1774191] +process[print_text-8]: started with pid [1774192] +process[rosbag_record-9]: started with pid [1774216] +started bagging! +preempting episode +[INFO] [1735708271.539539]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1735708272.332132]: Stored episode 3. +[INFO] [1735708272.339670]: Saving video... +[INFO] [1735708272.772218]: Video saved! +Starting episode 3 +[INFO] [1735708272.812567]: resetting environment +[INFO] [1735708272.838699]: cleared current plan +[INFO] [1735708272.839205]: moving to home +Move to target timeout: 5.100231647491455. Trans error: 0.005576091356133687. Rot error: 0.9949652568321697 +[INFO] [1735708283.649095]: reached home +[INFO] [1735708283.657415]: reset action history +[INFO] [1735708286.429991]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735708299.846536]: resetting environment +[INFO] [1735708299.864269]: cleared current plan +[INFO] [1735708299.864472]: moving to home +Move to target timeout: 5.002374649047852. Trans error: 0.005032249208118943. Rot error: 1.0296036885659667 +[INFO] [1735708305.871685]: reached home +[INFO] [1735708305.876530]: reset action history +[INFO] [1735708308.656034]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/episode_rosbags/episode_3_2025-01-01-00-11-48.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1773565.log +started roslaunch server http://158.130.50.37:38065/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1774367] +process[print_text-11]: started with pid [1774368] +process[rosbag_record-12]: started with pid [1774369] +started bagging! +preempting episode +[INFO] [1735708328.966325]: Storing episode... +[print_text-11] killing on exit +[pub_text-10] killing on exit +[rosbag_record-12] killing on exit +[INFO] [1735708329.751379]: Stored episode 4. +[INFO] [1735708329.757236]: Saving video... +[WARN] [1735708330.275734]: Plan exhausted +[WARN] [1735708330.317481]: Plan exhausted +[INFO] [1735708330.241907]: Video saved! +Starting episode 4 +[INFO] [1735708330.313069]: resetting environment +[INFO] [1735708330.348465]: cleared current plan +[INFO] [1735708330.348729]: moving to home +Move to target timeout: 5.002542018890381. Trans error: 0.006714248564094305. Rot error: 0.9052469680763231 +Move to target timeout: 5.095457553863525. Trans error: 0.0055652601040947925. Rot error: 1.0114760429812093 +[INFO] [1735708341.457429]: reached home +[INFO] [1735708341.468228]: reset action history +[INFO] [1735708344.257217]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735708357.121723]: resetting environment +[INFO] [1735708357.151000]: cleared current plan +[INFO] [1735708357.152947]: moving to home +[INFO] [1735708358.158409]: reached home +[INFO] [1735708358.167114]: reset action history +[INFO] [1735708360.944846]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/episode_rosbags/episode_4_2025-01-01-00-12-40.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1773565.log +started roslaunch server http://158.130.50.37:45805/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1774508] +process[print_text-14]: started with pid [1774509] +process[rosbag_record-15]: started with pid [1774510] +started bagging! +For topic gripper_width: timestamp difference is 114976761 for nearest: 1735708365919932482 - target: 1735708365804955721 at idx 73 +For topic gripper_width: timestamp difference is 114578153 for nearest: 1735708371653314258 - target: 1735708371538736105 at idx 145 +proceeding to start episode! +[INFO] [1735708395.851389]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735708396.691309]: Stored episode 5. +[INFO] [1735708396.693114]: Saving video... +[INFO] [1735708397.084822]: Video saved! +[INFO] [1735708397.159492]: resetting environment +[INFO] [1735708397.190319]: cleared current plan +[INFO] [1735708397.192329]: moving to home +Move to target timeout: 5.001818418502808. Trans error: 0.005475922953337431. Rot error: 0.765469132885039 +Move to target timeout: 5.0976855754852295. Trans error: 0.005344645620675619. Rot error: 0.8332777398040311 +[INFO] [1735708408.301337]: reached home +[INFO] [1735708408.306372]: reset action history +[INFO] [1735708411.112552]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/requirements.txt b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-metadata.json b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e2cba4f5826ed13f34879a0bb7a3f54707d695b3 --- /dev/null +++ b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T05:06:11.699195", + "startedAt": "2025-01-01T05:06:11.308943", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4123.861625, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.902, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4498.097, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4498.146, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.004, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 641.4450988769531 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..deb999a1803645fcaf383095c42f7d09015b5fad --- /dev/null +++ b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "ba360af7c8232a7b8ce8aaa030dd270211880cfd5b254c4bc04bf8a43662e7a2", "size": 1282700, "path": "media/videos/eval/0_eval_0_ba360af7c8232a7b8ce8.mp4"}, "global_step": 524, "_timestamp": 1735708421.6833305, "_runtime": 450.3660395145416, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "c7f634ec4b728ff21a6acbff209564236673efe05774bd33a9431331b430c7a1", "size": 1434311, "path": "media/videos/eval/1_eval_1_c7f634ec4b728ff21a6a.mp4"}, "eval/num_success": 0.0, "episode": 30.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "e83738fb6ffd08c2daf38b899a4b987a92c44d56b7a7e79502a841a7273f8c95", "size": 870326, "path": "media/videos/eval/2_eval_2_e83738fb6ffd08c2daf3.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "0390825e5beb348802816ef3517f3a8a887d453cb1d4be00750cfb7558d508ad", "size": 907684, "path": "media/videos/eval/3_eval_3_0390825e5beb34880281.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "c88ccf3ee860d0a1e490c8672df176570ddde0118f4d935801d393e7925c4286", "size": 1540490, "path": "media/videos/eval/4_eval_4_c88ccf3ee860d0a1e490.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://2dp0n6el38y2fz52js25ucdx5s7bh9k0f8h9sm7f6x5zsrnlaj0s5xa928vgbjjxi6bg32wu0qv0ow2elsfkhhhcisdqroqvifl5gllv5e32a466toq3qd9ej7soima5/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://39qcj4fpfz31000tlqox3dfs9mvg7077zooehm5f4h5ouhji90b5zxz5cx4zmf3ht80cnr34koj6yq2s21jsgjrx2xxnqdg3fq5l8x7emftd5u60ch1xyjoppmlbdst7:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 15, "total_num_episodes": 30, "total_success_rate": 0.5, "total_success_rate_lb_0.01": 0.2916622460305133, "total_success_rate_ub_0.01": 0.7063640355651114, "total_success_rate_lb_0.025": 0.33155784014255996, "total_success_rate_ub_0.025": 0.6857495158539253, "total_success_rate_lb_0.05": 0.35735600444547144, "total_success_rate_ub_0.05": 0.642512828083146, "total_success_rate_lb_0.1": 0.37515496092551415, "total_success_rate_ub_0.1": 0.6270253248258695, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "0ee4b68a560cd8157b36c2d1fd5d32fffc04bad468a8fb68639e23712beac32e", "size": 507, "artifact_path": "wandb-client-artifact://yl8i4rwyrq4b3iglvs4ziq3kfncmyi506c5bkuw1if3qkmgq5agauc15w61d1w5c2tc7eorl03eg9808zgjst0a4n1smrq2izcut5vtds74wjs1inkklpox0qut9rsa2/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://dmelfrctoc36zqyna60fq5nus7r8jg1cg5ukznjukqb6vyfiga4mni9u5m6to07u8sjkdusmvxteco07adauyfhm02dhqgsi644b7zqr6slsbirxy6yezajt8h5jrsvw:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_0ee4b68a560cd8157b36.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 450}, "eval/total_success_rate": 0.5, "eval/total_success_rate_lb_0.01": 0.2916622459888458, "eval/total_success_rate_ub_0.01": 0.7063640356063843, "eval/total_success_rate_lb_0.025": 0.33155784010887146, "eval/total_success_rate_ub_0.025": 0.6857495307922363, "eval/total_success_rate_lb_0.05": 0.3573560118675232, "eval/total_success_rate_ub_0.05": 0.6425127983093262, "eval/total_success_rate_lb_0.1": 0.375154972076416, "eval/total_success_rate_ub_0.1": 0.627025306224823} \ No newline at end of file diff --git a/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/logs/debug-internal.log b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f761ac903b31c879afd967b082d5778e40db6846 --- /dev/null +++ b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/logs/debug-internal.log @@ -0,0 +1,1031 @@ +2025-01-01 00:06:11,317 INFO StreamThr :1773737 [internal.py:wandb_internal():85] W&B internal server running at pid: 1773737, started at: 2025-01-01 00:06:11.317096 +2025-01-01 00:06:11,318 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status +2025-01-01 00:06:11,329 INFO WriterThread:1773737 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/run-7j9cp4xk.wandb +2025-01-01 00:06:11,330 DEBUG SenderThread:1773737 [sender.py:send():379] send: header +2025-01-01 00:06:11,338 DEBUG SenderThread:1773737 [sender.py:send():379] send: run +2025-01-01 00:06:11,569 INFO SenderThread:1773737 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files +2025-01-01 00:06:11,569 INFO SenderThread:1773737 [sender.py:_start_run_threads():1188] run started: 7j9cp4xk with start time 1735707971.317291 +2025-01-01 00:06:11,593 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 00:06:11,593 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: check_version +2025-01-01 00:06:11,681 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 00:06:11,686 DEBUG HandlerThread:1773737 [system_info.py:__init__():26] System info init +2025-01-01 00:06:11,686 DEBUG HandlerThread:1773737 [system_info.py:__init__():41] System info init done +2025-01-01 00:06:11,686 INFO HandlerThread:1773737 [system_monitor.py:start():194] Starting system monitor +2025-01-01 00:06:11,686 INFO SystemMonitor:1773737 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 00:06:11,686 INFO HandlerThread:1773737 [system_monitor.py:probe():214] Collecting system info +2025-01-01 00:06:11,687 INFO SystemMonitor:1773737 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 00:06:11,688 INFO SystemMonitor:1773737 [interfaces.py:start():188] Started disk monitoring +2025-01-01 00:06:11,688 INFO SystemMonitor:1773737 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 00:06:11,689 INFO SystemMonitor:1773737 [interfaces.py:start():188] Started memory monitoring +2025-01-01 00:06:11,689 INFO SystemMonitor:1773737 [interfaces.py:start():188] Started network monitoring +2025-01-01 00:06:11,699 DEBUG HandlerThread:1773737 [system_info.py:probe():152] Probing system +2025-01-01 00:06:11,700 DEBUG HandlerThread:1773737 [system_info.py:_probe_git():137] Probing git +2025-01-01 00:06:11,706 DEBUG HandlerThread:1773737 [system_info.py:_probe_git():145] Probing git done +2025-01-01 00:06:11,706 DEBUG HandlerThread:1773737 [system_info.py:probe():200] Probing system done +2025-01-01 00:06:11,706 DEBUG HandlerThread:1773737 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T05:06:11.699195', 'startedAt': '2025-01-01T05:06:11.308943', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4123.861625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.902, 'min': 800.0, 'max': 4700.0}, {'current': 4498.097, 'min': 800.0, 'max': 4700.0}, {'current': 4498.146, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.004, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 641.4450988769531}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 00:06:11,706 INFO HandlerThread:1773737 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 00:06:11,706 INFO HandlerThread:1773737 [system_monitor.py:probe():227] Publishing system info +2025-01-01 00:06:11,707 DEBUG HandlerThread:1773737 [system_info.py:_save_code():44] Saving code +2025-01-01 00:06:11,708 DEBUG HandlerThread:1773737 [system_info.py:_save_code():67] Saving code done +2025-01-01 00:06:11,708 DEBUG HandlerThread:1773737 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 00:06:11,771 DEBUG HandlerThread:1773737 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 00:06:11,771 INFO HandlerThread:1773737 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 00:06:11,774 DEBUG SenderThread:1773737 [sender.py:send():379] send: files +2025-01-01 00:06:11,774 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 00:06:11,774 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 00:06:11,775 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 00:06:12,014 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 00:06:12,014 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: python_packages +2025-01-01 00:06:12,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:06:12,016 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:06:12,025 INFO wandb-upload_2:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/gvrny1vq-diff.patch +2025-01-01 00:06:12,040 INFO wandb-upload_0:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/fd0e1vx3-wandb-metadata.json +2025-01-01 00:06:12,053 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/se8x2jqq-code/FISH/eval_robot.py +2025-01-01 00:06:12,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:12,057 INFO HandlerThread:1773737 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/tb" + save: true +} +control { + relay_id: "e089ddc1df5a4e7a9851f3eb0aa21762" +} +_info { + stream_id: "7j9cp4xk" +} + +2025-01-01 00:06:12,057 DEBUG HandlerThread:1773737 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 00:06:12,098 DEBUG SenderThread:1773737 [sender.py:send():379] send: telemetry +2025-01-01 00:06:12,108 DEBUG SenderThread:1773737 [sender.py:send():379] send: tbrecord +2025-01-01 00:06:12,109 DEBUG SenderThread:1773737 [sender.py:send():379] send: files +2025-01-01 00:06:12,109 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 with policy live +2025-01-01 00:06:12,291 INFO wandb-upload_2:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/edog103e-events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:06:12,571 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-metadata.json +2025-01-01 00:06:12,571 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:06:12,571 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:06:12,571 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/diff.patch +2025-01-01 00:06:12,571 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/code/FISH/eval_robot.py +2025-01-01 00:06:12,572 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/requirements.txt +2025-01-01 00:06:12,572 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/code +2025-01-01 00:06:12,572 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/code/FISH +2025-01-01 00:06:13,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:14,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:14,570 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:06:15,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:16,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:17,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:06:17,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:18,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:19,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:20,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:21,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:22,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:23,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:06:23,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:24,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:25,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:26,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:27,014 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:06:27,015 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:06:27,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:28,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:28,159 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:06:29,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:30,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:31,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:32,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:32,133 DEBUG SenderThread:1773737 [sender.py:send():379] send: config +2025-01-01 00:06:33,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:33,178 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:06:34,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:34,575 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:06:35,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:36,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:37,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:38,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:38,185 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:06:38,577 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:06:39,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:40,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:41,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:42,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:06:42,015 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:06:42,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:43,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:43,250 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:06:43,578 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/config.yaml +2025-01-01 00:06:44,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:44,578 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:06:45,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:46,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:47,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:48,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:48,579 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:06:49,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:06:49,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:50,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:51,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:52,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:52,580 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:06:53,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:54,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:54,581 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:06:54,682 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:06:55,020 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:56,020 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:56,582 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:06:57,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:06:57,015 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:06:57,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:58,020 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:06:59,020 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:00,020 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:00,148 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:01,020 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:02,020 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:03,020 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:04,020 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:05,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:06,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:06,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:07,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:08,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:09,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:10,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:11,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:11,366 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:11,690 DEBUG SystemMonitor:1773737 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 00:07:11,690 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:07:12,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:07:12,015 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:07:12,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:12,586 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:07:13,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:14,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:15,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:16,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:16,377 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:16,587 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:07:17,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:18,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:18,588 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:07:19,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:20,023 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:20,588 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:07:21,023 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:22,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:22,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:23,023 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:24,023 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:25,024 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:26,024 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:27,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:07:27,015 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:07:27,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:27,151 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:28,024 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:29,024 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:30,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:31,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:32,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:33,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:33,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:34,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:35,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:36,026 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:37,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:38,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:38,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:38,593 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:07:39,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:40,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:41,026 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:41,691 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:07:42,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:07:42,016 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:07:42,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:43,027 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:43,082 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:44,028 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:45,029 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:46,028 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:47,028 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:48,028 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:48,516 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:49,029 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:49,504 DEBUG SenderThread:1773737 [sender.py:send():379] send: files +2025-01-01 00:07:49,504 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_ba360af7c8232a7b8ce8.mp4 with policy now +2025-01-01 00:07:49,504 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:07:49,505 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:07:49,505 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:07:49,506 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:07:49,596 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:07:49,596 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/0_eval_0_ba360af7c8232a7b8ce8.mp4 +2025-01-01 00:07:49,596 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos +2025-01-01 00:07:49,596 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval +2025-01-01 00:07:49,596 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media +2025-01-01 00:07:49,753 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/4t98gag9-media/videos/eval/0_eval_0_ba360af7c8232a7b8ce8.mp4 +2025-01-01 00:07:50,004 INFO wandb-upload_2:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/4d6045ve-media/videos/eval/0_eval_0_ba360af7c8232a7b8ce8.mp4 +2025-01-01 00:07:50,029 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:50,597 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:07:51,029 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:52,029 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:53,029 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:54,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:07:54,030 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:55,030 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:56,030 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:56,599 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:07:57,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:07:57,016 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:07:57,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:58,030 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:59,030 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:07:59,153 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:00,031 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:00,601 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:01,031 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:02,031 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:02,601 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:03,031 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:04,031 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:04,373 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:04,602 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:05,032 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:06,032 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:07,032 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:08,032 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:09,032 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:10,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:10,032 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:11,032 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:11,691 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:08:12,015 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:08:12,016 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:08:12,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:12,605 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:08:12,846 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/mkn3t4aj-events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:08:13,033 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:14,033 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:14,606 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:15,033 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:15,172 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:16,033 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:17,033 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:18,034 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:18,607 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:19,034 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:20,034 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:20,608 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:20,791 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:21,036 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:22,036 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:22,608 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:23,036 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:24,036 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:25,036 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:26,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:26,037 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:27,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:08:27,016 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:08:27,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:28,037 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:29,037 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:30,037 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:31,038 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:31,100 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:32,037 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:33,037 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:34,037 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:35,037 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:36,037 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:37,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:37,038 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:38,038 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:39,038 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:40,038 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:41,038 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:41,692 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:08:42,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:08:42,016 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:08:42,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:42,167 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:42,615 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:43,039 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:44,039 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:45,039 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:46,039 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:47,039 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:48,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:48,039 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:49,040 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:50,040 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:51,041 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:52,041 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:53,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:53,041 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:54,041 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:55,041 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:56,041 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:56,620 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:57,021 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:08:57,021 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:08:57,043 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:57,122 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:08:57,125 DEBUG SenderThread:1773737 [sender.py:send():379] send: files +2025-01-01 00:08:57,125 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_c7f634ec4b728ff21a6a.mp4 with policy now +2025-01-01 00:08:57,125 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:08:57,126 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:08:57,126 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:08:57,401 INFO wandb-upload_2:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/urucpi4t-media/videos/eval/1_eval_1_c7f634ec4b728ff21a6a.mp4 +2025-01-01 00:08:57,620 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:08:57,621 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/1_eval_1_c7f634ec4b728ff21a6a.mp4 +2025-01-01 00:08:57,621 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval +2025-01-01 00:08:57,924 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/6gabbvs4-media/videos/eval/1_eval_1_c7f634ec4b728ff21a6a.mp4 +2025-01-01 00:08:58,048 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:58,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:08:58,621 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:08:59,045 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:08:59,621 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:09:00,050 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:01,048 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:02,049 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:03,046 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:03,197 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:04,050 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:04,623 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:09:05,051 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:06,053 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:07,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:08,053 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:08,209 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:08,624 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:09:09,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:10,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:10,625 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:09:11,051 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:11,692 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:09:12,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:09:12,018 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:09:12,065 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:12,626 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:09:13,051 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:14,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:14,150 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:15,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:16,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:17,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:18,053 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:19,053 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:20,029 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:20,053 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:21,054 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:22,053 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:23,053 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:24,053 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:25,055 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:26,028 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:26,055 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:27,016 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:09:27,017 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:09:27,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:28,054 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:29,054 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:30,054 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:31,055 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:31,082 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:32,054 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:33,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:34,056 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:35,055 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:36,055 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:37,028 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:37,055 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:38,055 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:39,055 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:40,055 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:41,056 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:41,693 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:09:42,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:09:42,022 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:09:42,065 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:42,170 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:43,056 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:44,056 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:45,058 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:46,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:47,056 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:48,031 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:48,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:49,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:50,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:51,057 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:52,059 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:53,060 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:54,030 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:09:54,058 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:55,058 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:56,059 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:57,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:09:57,017 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:09:57,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:58,059 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:59,060 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:09:59,164 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:00,059 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:01,060 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:02,060 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:03,063 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:04,060 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:05,030 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:05,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:06,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:07,062 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:08,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:09,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:10,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:11,031 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:11,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:11,693 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:10:12,017 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:10:12,017 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:10:12,065 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:13,061 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:14,063 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:15,062 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:16,062 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:16,175 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:17,062 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:18,062 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:19,065 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:20,062 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:21,064 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:22,031 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:22,063 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:23,063 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:24,063 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:25,063 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:26,064 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:27,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:10:27,018 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:10:27,063 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:27,079 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:28,063 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:29,066 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:30,064 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:31,065 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:32,066 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:33,032 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:33,065 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:34,066 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:35,066 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:36,066 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:37,068 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:38,033 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:38,066 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:39,066 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:40,066 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:41,067 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:41,694 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:10:42,019 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:10:42,019 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:10:42,067 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:43,067 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:43,173 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:43,655 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:10:43,909 INFO wandb-upload_2:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/3in8ytud-events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:10:44,067 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:44,646 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:10:44,646 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:10:44,646 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:10:44,656 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:10:44,656 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:10:45,068 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:46,068 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:47,068 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:48,072 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:48,508 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:49,071 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:50,068 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:50,658 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:10:51,069 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:52,069 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:53,069 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:54,069 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:54,504 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:10:54,659 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:10:55,069 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:56,069 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:57,018 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:10:57,018 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:10:57,069 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:58,070 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:10:59,069 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:00,070 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:00,104 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:01,070 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:02,070 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:03,070 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:04,070 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:05,071 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:06,034 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:06,071 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:07,071 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:08,074 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:09,074 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:10,074 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:11,074 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:11,537 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:11,694 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:11:12,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:11:12,025 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:11:12,076 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:12,665 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:11:12,805 DEBUG SenderThread:1773737 [sender.py:send():379] send: files +2025-01-01 00:11:12,806 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_e83738fb6ffd08c2daf3.mp4 with policy now +2025-01-01 00:11:12,806 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:11:12,807 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:11:12,807 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:11:12,809 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:11:13,055 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/2a211tmn-media/videos/eval/2_eval_2_e83738fb6ffd08c2daf3.mp4 +2025-01-01 00:11:13,077 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:13,666 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:11:13,666 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/2_eval_2_e83738fb6ffd08c2daf3.mp4 +2025-01-01 00:11:13,666 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval +2025-01-01 00:11:13,945 INFO wandb-upload_2:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/g5lih3yv-media/videos/eval/2_eval_2_e83738fb6ffd08c2daf3.mp4 +2025-01-01 00:11:14,078 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:14,666 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:11:15,080 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:16,089 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:16,839 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:17,092 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:18,090 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:19,091 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:20,091 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:21,091 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:22,042 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:22,100 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:23,093 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:24,095 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:24,669 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:11:25,097 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:26,098 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:27,023 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:11:27,024 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:11:27,097 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:27,138 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:28,100 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:28,671 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:11:29,110 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:30,105 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:31,113 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:32,110 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:33,040 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:33,110 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:34,111 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:35,113 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:36,114 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:37,111 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:38,041 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:38,117 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:39,117 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:40,113 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:40,496 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:11:40,497 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:11:40,497 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:11:40,674 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:11:40,675 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:11:40,675 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:11:41,121 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:41,694 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:11:42,025 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:11:42,026 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:11:42,118 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:43,118 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:43,125 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:44,117 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:45,118 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:46,118 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:46,677 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:11:47,118 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:48,121 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:48,657 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:49,119 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:50,121 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:50,678 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:11:51,121 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:52,121 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:52,679 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:11:53,121 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:54,043 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:11:54,121 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:55,122 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:56,122 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:57,022 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:11:57,022 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:11:57,122 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:58,122 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:59,123 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:11:59,132 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:00,123 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:01,124 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:02,124 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:03,124 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:04,124 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:05,043 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:05,124 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:06,124 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:07,124 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:08,124 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:09,125 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:10,137 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:10,245 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:10,301 DEBUG SenderThread:1773737 [sender.py:send():379] send: files +2025-01-01 00:12:10,301 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_0390825e5beb34880281.mp4 with policy now +2025-01-01 00:12:10,303 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:12:10,304 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:12:10,304 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:12:10,322 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:12:10,572 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/ywpogo1l-media/videos/eval/3_eval_3_0390825e5beb34880281.mp4 +2025-01-01 00:12:10,685 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:12:10,685 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:10,685 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/3_eval_3_0390825e5beb34880281.mp4 +2025-01-01 00:12:10,685 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval +2025-01-01 00:12:10,959 INFO wandb-upload_2:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/z7yvmmtp-media/videos/eval/3_eval_3_0390825e5beb34880281.mp4 +2025-01-01 00:12:11,147 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:11,685 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:11,695 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:12:12,033 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:12:12,033 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:12:12,149 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:12,686 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:13,143 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:14,145 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:15,147 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:15,352 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:16,149 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:16,687 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:17,147 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:18,152 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:19,150 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:20,149 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:20,452 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:21,153 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:22,159 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:22,689 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:23,153 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:24,157 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:25,155 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:26,156 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:26,309 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:26,690 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:27,029 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:12:27,029 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:12:27,154 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:28,157 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:29,153 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:30,157 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:31,155 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:32,058 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:32,160 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:33,159 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:34,165 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:35,164 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:36,161 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:37,059 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:37,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:37,573 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:12:37,574 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:12:37,574 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:12:37,694 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:12:37,694 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:12:37,937 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/ijntgok8-events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:12:38,201 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:38,694 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:39,177 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:40,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:40,695 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:41,168 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:41,696 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:12:42,024 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:12:42,024 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:12:42,168 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:42,177 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:42,697 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:43,168 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:44,168 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:45,168 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:46,168 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:46,698 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:47,168 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:48,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:48,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:49,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:50,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:51,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:52,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:52,700 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:12:53,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:54,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:54,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:55,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:56,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:57,024 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:12:57,024 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:12:57,170 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:58,170 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:12:59,148 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:12:59,170 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:00,170 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:01,170 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:02,171 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:03,171 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:04,171 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:05,052 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:05,172 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:06,171 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:07,172 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:08,172 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:09,172 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:10,172 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:10,707 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:10,905 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:11,174 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:11,696 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:13:12,024 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:13:12,024 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:13:12,172 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:13,172 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:14,172 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:15,173 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:15,936 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:16,187 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:16,709 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:17,157 DEBUG SenderThread:1773737 [sender.py:send():379] send: files +2025-01-01 00:13:17,158 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_c88ccf3ee860d0a1e490.mp4 with policy now +2025-01-01 00:13:17,158 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:13:17,158 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:13:17,159 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:17,171 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:17,178 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:17,419 INFO wandb-upload_2:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/u5m8ksmp-media/videos/eval/4_eval_4_c88ccf3ee860d0a1e490.mp4 +2025-01-01 00:13:17,709 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:13:17,709 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/4_eval_4_c88ccf3ee860d0a1e490.mp4 +2025-01-01 00:13:17,709 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval +2025-01-01 00:13:17,993 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/0zgzs2oo-media/videos/eval/4_eval_4_c88ccf3ee860d0a1e490.mp4 +2025-01-01 00:13:18,178 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:18,709 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:19,185 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:20,188 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:21,183 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:21,234 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:22,184 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:23,194 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:24,191 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:24,711 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:25,196 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:26,193 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:26,237 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:27,030 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:13:27,031 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:13:27,197 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:28,198 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:28,713 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:29,210 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:30,207 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:30,714 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:31,204 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:32,076 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:32,207 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:32,714 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:33,211 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:34,227 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:35,218 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:36,220 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:37,076 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:37,220 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:38,223 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:39,217 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:40,229 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:41,150 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 00:13:41,150 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 00:13:41,234 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:41,367 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:13:41,611 INFO wandb-upload_2:1773737 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpe27epawh +2025-01-01 00:13:41,717 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:13:41,718 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 00:13:41,718 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media +2025-01-01 00:13:41,718 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table/eval +2025-01-01 00:13:41,718 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table +2025-01-01 00:13:41,773 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 00:13:41,783 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:13:41,983 INFO SenderThread:1773737 [sender.py:send_request_log_artifact():1525] logged artifact run-7j9cp4xk-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIyODI1Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE4MTA3Mw==', 'latestArtifact': None}} +2025-01-01 00:13:41,983 DEBUG SenderThread:1773737 [sender.py:send():379] send: files +2025-01-01 00:13:41,983 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-01 00:13:41,985 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:13:41,985 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,985 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,985 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:13:41,986 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,986 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,986 DEBUG SenderThread:1773737 [sender.py:send():379] send: telemetry +2025-01-01 00:13:41,986 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,986 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,986 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,987 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,987 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,987 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,988 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,988 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,988 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,988 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,988 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,989 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,989 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,989 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,989 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,990 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,990 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,990 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,990 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,991 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,991 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:41,991 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:41,991 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:13:41,991 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 00:13:42,024 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:13:42,206 INFO wandb-upload_0:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/3h0auid4-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 00:13:42,218 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:42,222 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/v1tu7egy-events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:13:42,386 INFO wandb-upload_2:1773737 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjh910wfh +2025-01-01 00:13:42,466 INFO wandb-upload_0:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/rkwxvp6d-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 00:13:42,718 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:13:42,718 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:42,718 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table/eval/total_success_rate_ci_6_0ee4b68a560cd8157b36.table.json +2025-01-01 00:13:42,718 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table/eval +2025-01-01 00:13:42,799 INFO SenderThread:1773737 [sender.py:send_request_log_artifact():1525] logged artifact run-7j9cp4xk-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIyODI2Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE4MTA3NA==', 'latestArtifact': None}} +2025-01-01 00:13:42,799 DEBUG SenderThread:1773737 [sender.py:send():379] send: files +2025-01-01 00:13:42,799 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_0ee4b68a560cd8157b36.table.json with policy now +2025-01-01 00:13:42,800 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:42,801 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:13:42,801 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:42,801 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:42,801 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:13:42,878 DEBUG SenderThread:1773737 [sender.py:send():379] send: exit +2025-01-01 00:13:42,878 INFO SenderThread:1773737 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 00:13:42,878 INFO SenderThread:1773737 [sender.py:send_exit():588] handling runtime: 450 +2025-01-01 00:13:42,878 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:42,878 INFO SenderThread:1773737 [sender.py:send_exit():594] send defer +2025-01-01 00:13:42,879 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:42,879 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 00:13:42,879 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:42,879 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 00:13:42,879 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 1 +2025-01-01 00:13:42,879 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:42,879 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 00:13:42,879 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:42,879 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 00:13:42,879 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 2 +2025-01-01 00:13:42,879 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:42,879 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 00:13:42,879 INFO HandlerThread:1773737 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 00:13:42,880 DEBUG SystemMonitor:1773737 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 00:13:42,880 DEBUG SystemMonitor:1773737 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 00:13:42,880 INFO HandlerThread:1773737 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 00:13:42,880 INFO HandlerThread:1773737 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 00:13:42,932 INFO HandlerThread:1773737 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 00:13:42,933 INFO HandlerThread:1773737 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 00:13:42,933 INFO HandlerThread:1773737 [interfaces.py:finish():200] Joined network monitor +2025-01-01 00:13:42,933 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:42,933 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 00:13:42,933 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 3 +2025-01-01 00:13:42,933 DEBUG SenderThread:1773737 [sender.py:send():379] send: stats +2025-01-01 00:13:42,934 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:42,934 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 00:13:42,934 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:42,934 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 00:13:42,934 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 4 +2025-01-01 00:13:42,934 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:42,934 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 00:13:42,995 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/6y9cw5e3-media/table/eval/total_success_rate_ci_6_0ee4b68a560cd8157b36.table.json +2025-01-01 00:13:43,270 INFO wandb-upload_0:1773737 [upload_job.py:push():130] Uploaded file /tmp/tmp46gdf0zrwandb/gv4395un-media/table/eval/total_success_rate_ci_6_0ee4b68a560cd8157b36.table.json +2025-01-01 00:13:43,718 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:13:44,719 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:48,720 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/config.yaml +2025-01-01 00:13:49,644 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:13:49,644 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:49,644 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 00:13:49,644 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:49,646 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 00:13:49,646 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 5 +2025-01-01 00:13:49,646 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:13:49,646 DEBUG SenderThread:1773737 [sender.py:send():379] send: history +2025-01-01 00:13:49,646 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:13:49,646 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:49,646 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 00:13:49,647 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:49,647 DEBUG SenderThread:1773737 [sender.py:send():379] send: summary +2025-01-01 00:13:49,651 INFO SenderThread:1773737 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:13:49,651 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:49,651 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 00:13:49,651 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 6 +2025-01-01 00:13:49,651 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:49,652 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 00:13:49,652 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:49,652 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 00:13:49,652 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 7 +2025-01-01 00:13:49,652 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:13:49,652 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:49,652 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 00:13:49,652 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:49,652 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 00:13:49,721 INFO Thread-12 :1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:13:50,555 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:13:51,251 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 8 +2025-01-01 00:13:51,251 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:13:51,252 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:51,252 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 00:13:51,252 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:51,252 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 00:13:51,252 INFO SenderThread:1773737 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 00:13:51,254 INFO SenderThread:1773737 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 00:13:51,256 INFO SenderThread:1773737 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 00:13:51,269 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 9 +2025-01-01 00:13:51,269 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:51,270 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 00:13:51,270 DEBUG SenderThread:1773737 [sender.py:send():379] send: artifact +2025-01-01 00:13:51,474 INFO SenderThread:1773737 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjk2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjk2Ng==', 'versionIndex': 403}}} +2025-01-01 00:13:51,474 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:51,474 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 00:13:51,474 INFO SenderThread:1773737 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 00:13:51,555 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:13:51,722 INFO SenderThread:1773737 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:51,722 INFO SenderThread:1773737 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files +2025-01-01 00:13:51,722 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/config.yaml config.yaml +2025-01-01 00:13:51,722 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json wandb-summary.json +2025-01-01 00:13:51,722 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/requirements.txt requirements.txt +2025-01-01 00:13:51,722 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-metadata.json wandb-metadata.json +2025-01-01 00:13:51,723 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 events.out.tfevents.1735707972.leonmkim-ROG-Strix-G15CS-G15CS.1773565.0 +2025-01-01 00:13:51,723 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log output.log +2025-01-01 00:13:51,723 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/diff.patch diff.patch +2025-01-01 00:13:51,729 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 00:13:51,729 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/2_eval_2_e83738fb6ffd08c2daf3.mp4 media/videos/eval/2_eval_2_e83738fb6ffd08c2daf3.mp4 +2025-01-01 00:13:51,729 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/4_eval_4_c88ccf3ee860d0a1e490.mp4 media/videos/eval/4_eval_4_c88ccf3ee860d0a1e490.mp4 +2025-01-01 00:13:51,729 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/3_eval_3_0390825e5beb34880281.mp4 media/videos/eval/3_eval_3_0390825e5beb34880281.mp4 +2025-01-01 00:13:51,729 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/1_eval_1_c7f634ec4b728ff21a6a.mp4 media/videos/eval/1_eval_1_c7f634ec4b728ff21a6a.mp4 +2025-01-01 00:13:51,729 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/videos/eval/0_eval_0_ba360af7c8232a7b8ce8.mp4 media/videos/eval/0_eval_0_ba360af7c8232a7b8ce8.mp4 +2025-01-01 00:13:51,729 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table/eval/total_success_rate_ci_6_0ee4b68a560cd8157b36.table.json media/table/eval/total_success_rate_ci_6_0ee4b68a560cd8157b36.table.json +2025-01-01 00:13:51,729 INFO SenderThread:1773737 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 00:13:51,729 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 10 +2025-01-01 00:13:51,730 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:13:51,730 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:51,730 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 00:13:51,730 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:51,730 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 00:13:51,730 INFO SenderThread:1773737 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 00:13:51,911 INFO wandb-upload_1:1773737 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/wandb-summary.json +2025-01-01 00:13:51,923 INFO wandb-upload_3:1773737 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/config.yaml +2025-01-01 00:13:51,947 INFO wandb-upload_2:1773737 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/requirements.txt +2025-01-01 00:13:51,968 INFO wandb-upload_0:1773737 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/files/output.log +2025-01-01 00:13:52,168 INFO Thread-11 (_thread_body):1773737 [sender.py:transition_state():614] send defer: 11 +2025-01-01 00:13:52,169 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:52,169 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 00:13:52,169 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:52,169 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 00:13:52,169 INFO SenderThread:1773737 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 00:13:52,170 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 12 +2025-01-01 00:13:52,170 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:52,170 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 00:13:52,171 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:52,171 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 00:13:52,171 INFO SenderThread:1773737 [file_stream.py:finish():601] file stream finish called +2025-01-01 00:13:52,373 INFO SenderThread:1773737 [file_stream.py:finish():605] file stream finish is done +2025-01-01 00:13:52,373 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 13 +2025-01-01 00:13:52,373 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:52,373 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 00:13:52,373 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:52,373 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 00:13:52,373 INFO SenderThread:1773737 [sender.py:transition_state():614] send defer: 14 +2025-01-01 00:13:52,373 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:13:52,374 INFO HandlerThread:1773737 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 00:13:52,374 DEBUG SenderThread:1773737 [sender.py:send():379] send: final +2025-01-01 00:13:52,374 DEBUG SenderThread:1773737 [sender.py:send():379] send: footer +2025-01-01 00:13:52,374 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: defer +2025-01-01 00:13:52,374 INFO SenderThread:1773737 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 00:13:52,374 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:13:52,375 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:13:52,375 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 00:13:52,375 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: server_info +2025-01-01 00:13:52,376 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:13:52,377 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 00:13:52,377 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 00:13:52,378 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:13:52,422 DEBUG SenderThread:1773737 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:13:52,422 INFO MainThread:1773737 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 00:13:52,423 INFO MainThread:1773737 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 00:13:52,423 INFO MainThread:1773737 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 00:13:52,423 DEBUG HandlerThread:1773737 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 00:13:52,423 INFO HandlerThread:1773737 [handler.py:finish():882] shutting down handler +2025-01-01 00:13:53,377 INFO WriterThread:1773737 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/run-7j9cp4xk.wandb +2025-01-01 00:13:53,422 INFO SenderThread:1773737 [sender.py:finish():1615] shutting down sender +2025-01-01 00:13:53,422 INFO SenderThread:1773737 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 00:13:53,422 INFO SenderThread:1773737 [file_pusher.py:join():175] waiting for file pusher diff --git a/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/logs/debug.log b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..70fedefef334092b98fdb5017db2fa137c87d337 --- /dev/null +++ b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_setup.py:_flush():76] Configure stats pid to 1773565 +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/settings +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/logs/debug.log +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/logs/debug-internal.log +2025-01-01 00:06:11,311 INFO MainThread:1773565 [wandb_init.py:init():569] calling init triggers +2025-01-01 00:06:11,312 INFO MainThread:1773565 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk'} +2025-01-01 00:06:11,312 INFO MainThread:1773565 [wandb_init.py:init():619] starting backend +2025-01-01 00:06:11,312 INFO MainThread:1773565 [wandb_init.py:init():623] setting up manager +2025-01-01 00:06:11,316 INFO MainThread:1773565 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 00:06:11,317 INFO MainThread:1773565 [wandb_init.py:init():631] backend started and connected +2025-01-01 00:06:11,329 INFO MainThread:1773565 [wandb_init.py:init():720] updated telemetry +2025-01-01 00:06:11,336 INFO MainThread:1773565 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 00:06:11,593 INFO MainThread:1773565 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 00:06:11,631 INFO MainThread:1773565 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 00:06:11,632 INFO MainThread:1773565 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 00:06:12,015 INFO MainThread:1773565 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 00:06:12,015 INFO MainThread:1773565 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 00:06:12,015 INFO MainThread:1773565 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 00:06:12,016 INFO MainThread:1773565 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 00:06:12,017 INFO MainThread:1773565 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 00:06:12,017 INFO MainThread:1773565 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/7j9cp4xk/tb, True +2025-01-01 00:06:32,133 INFO MainThread:1773565 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-01 00:13:53,424 WARNING MsgRouterThr:1773565 [router.py:message_loop():77] message_loop has been closed diff --git a/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/run-7j9cp4xk.wandb b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/run-7j9cp4xk.wandb new file mode 100644 index 0000000000000000000000000000000000000000..61ba8fdb676839761271386eb7550500e7d6d028 Binary files /dev/null and b/7j9cp4xk/wandb/run-20250101_000611-7j9cp4xk/run-7j9cp4xk.wandb differ diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae92c4e2d5e593a3852a38e8582e2fe346700ff3 --- /dev/null +++ b/config.yaml @@ -0,0 +1,546 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '1006_0' +wandb_notes: 1006_0_req_1352_0restarted_2 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: true +use_sdf_maps: true +use_normals_maps: true +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 7 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/iz44h5u3/.hydra/config.yaml b/iz44h5u3/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05485f98b3c08bece875a460fe25a0cce7504f90 --- /dev/null +++ b/iz44h5u3/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1006_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/iz44h5u3/.hydra/hydra.yaml b/iz44h5u3/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a10566792d069d2eb38fa43064b4b2f12b6e0d3 --- /dev/null +++ b/iz44h5u3/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/iz44h5u3/.hydra/overrides.yaml b/iz44h5u3/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/iz44h5u3/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/iz44h5u3/episode_rosbags/aligned_depth_to_color_K.npy b/iz44h5u3/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/iz44h5u3/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/iz44h5u3/episode_rosbags/cam_tf_world.npy b/iz44h5u3/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/iz44h5u3/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/iz44h5u3/episode_rosbags/color_K.npy b/iz44h5u3/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/iz44h5u3/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/iz44h5u3/episode_rosbags/depth_K.npy b/iz44h5u3/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/iz44h5u3/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/iz44h5u3/eval_robot.log b/iz44h5u3/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..566a8761afd082df840a759bafd5ed6f12b77fef --- /dev/null +++ b/iz44h5u3/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-01 11:52:07,015][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 11:52:07,018][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/iz44h5u3/eval_video/0_eval.mp4 b/iz44h5u3/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2d52f7c887dc97ca5e162811f9d0dc2a47dff1fe --- /dev/null +++ b/iz44h5u3/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e9e62ad5c72d29c1b12983b1c5060a80faaef19213c585671a0d6e714502baa +size 1474906 diff --git a/iz44h5u3/eval_video/1_eval.mp4 b/iz44h5u3/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..76b03ae2aa87d6b9e7b7058bebae65f306daba4b --- /dev/null +++ b/iz44h5u3/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e1d3d257c24ab6b47342a8b2ccd131cace40a2db1f892f2fae54d5cac392913 +size 1448243 diff --git a/iz44h5u3/eval_video/2_eval.mp4 b/iz44h5u3/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..69d9e91d73e2d548635dbf37034297367eafb3aa --- /dev/null +++ b/iz44h5u3/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adc4ed4f260e51275bb5d919665cd0984053c6bf51e70a2f97b7ae22703a0173 +size 1444569 diff --git a/iz44h5u3/eval_video/3_eval.mp4 b/iz44h5u3/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..98e2536654ee0b3c800fbd3e46c093e0ef6c1002 --- /dev/null +++ b/iz44h5u3/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d085060cf5254eaf2ca93f0f171c3c8434e038096a28844daee2e3c33dcbc6b +size 1448998 diff --git a/iz44h5u3/eval_video/4_eval.mp4 b/iz44h5u3/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..69b27bbaaed68b2aeed18526da7b11261481413b --- /dev/null +++ b/iz44h5u3/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:faf38f4497652bfc9799e67edee1d3db615a4c248415463093005d199797907e +size 1436685 diff --git a/iz44h5u3/tb/events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 b/iz44h5u3/tb/events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 new file mode 100644 index 0000000000000000000000000000000000000000..eaa3db414127aeb979b6a95e7832d889ad70ff0d --- /dev/null +++ b/iz44h5u3/tb/events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f07585f68de480dadd498278f811b1a6c4a4800898736ed933699e7565b3f33c +size 6539 diff --git a/iz44h5u3/wandb/debug-internal.log b/iz44h5u3/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6fd48134700f66b794d0c3705439ca13eb062c3c --- /dev/null +++ b/iz44h5u3/wandb/debug-internal.log @@ -0,0 +1,846 @@ +2025-01-01 11:52:13,003 INFO StreamThr :1799171 [internal.py:wandb_internal():85] W&B internal server running at pid: 1799171, started at: 2025-01-01 11:52:13.003247 +2025-01-01 11:52:13,004 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status +2025-01-01 11:52:13,015 INFO WriterThread:1799171 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/run-iz44h5u3.wandb +2025-01-01 11:52:13,015 DEBUG SenderThread:1799171 [sender.py:send():379] send: header +2025-01-01 11:52:13,023 DEBUG SenderThread:1799171 [sender.py:send():379] send: run +2025-01-01 11:52:13,261 INFO SenderThread:1799171 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files +2025-01-01 11:52:13,262 INFO SenderThread:1799171 [sender.py:_start_run_threads():1188] run started: iz44h5u3 with start time 1735750333.003159 +2025-01-01 11:52:13,279 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 11:52:13,279 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: check_version +2025-01-01 11:52:13,351 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 11:52:13,355 DEBUG HandlerThread:1799171 [system_info.py:__init__():26] System info init +2025-01-01 11:52:13,356 DEBUG HandlerThread:1799171 [system_info.py:__init__():41] System info init done +2025-01-01 11:52:13,356 INFO HandlerThread:1799171 [system_monitor.py:start():194] Starting system monitor +2025-01-01 11:52:13,356 INFO SystemMonitor:1799171 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 11:52:13,356 INFO HandlerThread:1799171 [system_monitor.py:probe():214] Collecting system info +2025-01-01 11:52:13,356 INFO SystemMonitor:1799171 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 11:52:13,357 INFO SystemMonitor:1799171 [interfaces.py:start():188] Started disk monitoring +2025-01-01 11:52:13,357 INFO SystemMonitor:1799171 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 11:52:13,358 INFO SystemMonitor:1799171 [interfaces.py:start():188] Started memory monitoring +2025-01-01 11:52:13,358 INFO SystemMonitor:1799171 [interfaces.py:start():188] Started network monitoring +2025-01-01 11:52:13,369 DEBUG HandlerThread:1799171 [system_info.py:probe():152] Probing system +2025-01-01 11:52:13,371 DEBUG HandlerThread:1799171 [system_info.py:_probe_git():137] Probing git +2025-01-01 11:52:13,376 DEBUG HandlerThread:1799171 [system_info.py:_probe_git():145] Probing git done +2025-01-01 11:52:13,376 DEBUG HandlerThread:1799171 [system_info.py:probe():200] Probing system done +2025-01-01 11:52:13,376 DEBUG HandlerThread:1799171 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T16:52:13.369332', 'startedAt': '2025-01-01T16:52:12.992506', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3936.501125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.978, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.028, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 730.6149291992188}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 11:52:13,376 INFO HandlerThread:1799171 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 11:52:13,376 INFO HandlerThread:1799171 [system_monitor.py:probe():227] Publishing system info +2025-01-01 11:52:13,376 DEBUG HandlerThread:1799171 [system_info.py:_save_code():44] Saving code +2025-01-01 11:52:13,377 DEBUG HandlerThread:1799171 [system_info.py:_save_code():67] Saving code done +2025-01-01 11:52:13,377 DEBUG HandlerThread:1799171 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 11:52:13,440 DEBUG HandlerThread:1799171 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 11:52:13,440 INFO HandlerThread:1799171 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 11:52:13,443 DEBUG SenderThread:1799171 [sender.py:send():379] send: files +2025-01-01 11:52:13,443 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 11:52:13,444 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 11:52:13,444 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 11:52:13,688 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 11:52:13,688 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: python_packages +2025-01-01 11:52:13,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:52:13,689 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:52:13,692 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/qgxls3bd-wandb-metadata.json +2025-01-01 11:52:13,704 INFO wandb-upload_2:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/pj5o5gnf-diff.patch +2025-01-01 11:52:13,733 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:13,733 INFO HandlerThread:1799171 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/tb" + save: true +} +control { + relay_id: "11eed6c934be49448399bbc4ba5ead8a" +} +_info { + stream_id: "iz44h5u3" +} + +2025-01-01 11:52:13,733 DEBUG HandlerThread:1799171 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 11:52:13,759 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/ay0ft0ay-code/FISH/eval_robot.py +2025-01-01 11:52:13,838 DEBUG SenderThread:1799171 [sender.py:send():379] send: telemetry +2025-01-01 11:52:13,838 DEBUG SenderThread:1799171 [sender.py:send():379] send: tbrecord +2025-01-01 11:52:13,839 DEBUG SenderThread:1799171 [sender.py:send():379] send: files +2025-01-01 11:52:13,839 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 with policy live +2025-01-01 11:52:14,029 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/yn6p74yh-events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:52:14,263 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:52:14,263 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/diff.patch +2025-01-01 11:52:14,263 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/requirements.txt +2025-01-01 11:52:14,264 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/code/FISH/eval_robot.py +2025-01-01 11:52:14,264 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-metadata.json +2025-01-01 11:52:14,264 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:52:14,264 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/code +2025-01-01 11:52:14,264 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/code/FISH +2025-01-01 11:52:14,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:15,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:16,262 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:52:16,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:17,629 DEBUG SenderThread:1799171 [sender.py:send():379] send: config +2025-01-01 11:52:17,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:18,637 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:18,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:19,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:20,263 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:52:20,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:21,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:22,264 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:52:22,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:23,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:23,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:24,265 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:52:24,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:25,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:26,265 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:52:26,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:27,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:28,688 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:52:28,689 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:52:28,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:28,801 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:29,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:30,266 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:52:30,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:31,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:32,267 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:52:32,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:33,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:34,268 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:52:34,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:34,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:35,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:36,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:37,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:38,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:39,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:40,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:40,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:41,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:42,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:43,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:52:43,689 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:52:43,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:44,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:45,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:45,827 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:46,271 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/config.yaml +2025-01-01 11:52:46,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:47,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:48,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:49,693 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:50,693 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:51,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:51,693 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:52,693 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:53,693 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:54,693 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:55,694 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:56,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:56,694 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:57,694 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:58,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:52:58,689 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:52:58,733 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:59,694 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:00,694 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:01,694 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:01,829 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:02,695 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:03,695 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:04,695 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:05,695 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:06,695 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:07,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:07,695 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:08,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:08,845 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:53:08,846 DEBUG SenderThread:1799171 [sender.py:send():379] send: files +2025-01-01 11:53:08,846 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_8e9e62ad5c72d29c1b12.mp4 with policy now +2025-01-01 11:53:08,846 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:53:08,847 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:53:08,848 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:53:09,100 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/0mpf8g7o-media/videos/eval/0_eval_0_8e9e62ad5c72d29c1b12.mp4 +2025-01-01 11:53:09,277 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/0_eval_0_8e9e62ad5c72d29c1b12.mp4 +2025-01-01 11:53:09,277 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:53:09,277 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos +2025-01-01 11:53:09,277 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media +2025-01-01 11:53:09,278 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval +2025-01-01 11:53:09,546 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/cgzcydm9-media/videos/eval/0_eval_0_8e9e62ad5c72d29c1b12.mp4 +2025-01-01 11:53:09,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:10,277 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:53:10,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:11,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:12,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:12,837 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:13,359 DEBUG SystemMonitor:1799171 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 11:53:13,359 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:53:13,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:53:13,690 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:53:13,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:14,279 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:53:14,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:15,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:16,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:17,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:18,280 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:53:18,590 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:18,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:19,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:20,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:21,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:22,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:23,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:23,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:24,698 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:25,698 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:26,698 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:27,698 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:28,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:53:28,689 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:53:28,698 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:28,778 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:29,284 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:53:29,536 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/bpva2uaz-events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:53:29,698 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:30,284 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:53:30,698 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:31,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:32,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:33,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:34,285 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:53:34,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:34,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:35,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:36,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:37,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:38,287 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:53:38,700 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:39,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:39,700 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:40,700 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:41,700 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:42,700 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:43,360 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:53:43,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:53:43,690 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:53:43,737 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:44,701 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:44,799 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:45,701 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:46,701 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:47,701 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:48,290 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:53:48,701 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:49,701 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:50,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:50,701 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:51,702 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:52,291 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:53:52,702 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:53,703 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:54,703 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:55,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:55,703 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:56,703 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:57,703 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:58,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:53:58,690 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:53:58,733 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:59,703 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:00,704 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:00,782 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:01,704 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:02,704 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:03,704 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:04,705 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:05,705 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:06,396 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:06,705 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:07,708 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:07,858 DEBUG SenderThread:1799171 [sender.py:send():379] send: files +2025-01-01 11:54:07,858 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_9e1d3d257c24ab6b4734.mp4 with policy now +2025-01-01 11:54:07,860 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:54:07,861 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:54:07,861 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:54:07,862 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:54:08,119 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/ghelnd32-media/videos/eval/1_eval_1_9e1d3d257c24ab6b4734.mp4 +2025-01-01 11:54:08,296 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:54:08,297 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:54:08,297 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/1_eval_1_9e1d3d257c24ab6b4734.mp4 +2025-01-01 11:54:08,297 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval +2025-01-01 11:54:08,603 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/9hzuoahb-media/videos/eval/1_eval_1_9e1d3d257c24ab6b4734.mp4 +2025-01-01 11:54:08,709 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:09,711 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:10,298 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:54:10,708 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:11,708 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:11,712 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:12,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:13,360 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:54:13,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:54:13,691 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:54:13,737 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:14,299 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:54:14,711 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:15,710 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:16,300 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:54:16,711 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:16,766 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:17,711 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:18,711 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:19,711 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:20,711 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:21,712 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:22,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:22,712 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:23,712 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:24,712 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:25,304 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:54:25,712 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:26,169 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:54:26,169 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:54:26,169 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:54:26,305 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:54:26,305 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:54:26,712 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:27,712 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:28,177 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:28,306 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:54:28,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:54:28,690 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:54:28,733 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:29,713 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:30,306 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:54:30,713 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:31,713 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:32,713 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:33,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:33,713 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:34,713 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:35,713 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:36,308 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:54:36,714 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:37,714 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:38,309 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:54:38,714 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:39,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:39,714 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:40,714 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:41,714 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:42,714 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:43,361 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:54:43,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:54:43,691 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:54:43,733 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:44,715 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:44,812 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:45,715 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:46,715 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:47,715 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:48,715 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:49,715 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:50,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:50,715 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:51,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:52,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:53,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:54,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:55,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:55,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:56,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:57,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:58,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:54:58,691 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:54:58,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:59,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:00,316 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:55:00,717 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:00,837 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:01,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:02,717 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:03,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:04,717 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:05,037 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:55:05,037 DEBUG SenderThread:1799171 [sender.py:send():379] send: files +2025-01-01 11:55:05,038 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_adc4ed4f260e51275bb5.mp4 with policy now +2025-01-01 11:55:05,038 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:55:05,038 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:55:05,041 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:55:05,302 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/mnsqe9em-media/videos/eval/2_eval_2_adc4ed4f260e51275bb5.mp4 +2025-01-01 11:55:05,318 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:55:05,318 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/2_eval_2_adc4ed4f260e51275bb5.mp4 +2025-01-01 11:55:05,318 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval +2025-01-01 11:55:05,582 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/v90w54po-media/videos/eval/2_eval_2_adc4ed4f260e51275bb5.mp4 +2025-01-01 11:55:05,723 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:06,058 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:06,318 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:55:06,722 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:07,723 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:08,723 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:09,722 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:10,319 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:55:10,725 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:11,105 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:11,725 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:12,726 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:13,361 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:55:13,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:55:13,691 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:55:13,733 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:14,321 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:55:14,728 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:15,004 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:55:15,004 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:55:15,004 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:55:15,321 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:55:15,321 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:55:15,606 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/murc48he-events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:55:15,727 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:16,321 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:55:16,585 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:16,731 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:17,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:18,322 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:55:18,726 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:19,726 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:20,323 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:55:20,727 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:21,698 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:21,727 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:22,727 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:23,727 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:24,727 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:25,727 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:26,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:26,727 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:27,727 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:28,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:55:28,691 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:55:28,733 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:29,728 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:30,728 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:31,728 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:31,839 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:32,728 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:33,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:34,728 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:35,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:36,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:37,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:37,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:38,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:39,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:40,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:41,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:42,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:42,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:43,361 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:55:43,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:55:43,691 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:55:43,733 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:44,730 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:45,730 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:46,730 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:47,730 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:47,779 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:48,730 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:49,730 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:50,730 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:51,731 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:52,731 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:53,020 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:53,735 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:54,181 DEBUG SenderThread:1799171 [sender.py:send():379] send: files +2025-01-01 11:55:54,181 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_3d085060cf5254eaf2ca.mp4 with policy now +2025-01-01 11:55:54,192 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:55:54,192 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:55:54,193 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:55:54,201 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:55:54,334 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:55:54,334 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:55:54,334 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/3_eval_3_3d085060cf5254eaf2ca.mp4 +2025-01-01 11:55:54,334 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval +2025-01-01 11:55:54,435 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/71mdrzum-media/videos/eval/3_eval_3_3d085060cf5254eaf2ca.mp4 +2025-01-01 11:55:54,696 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/v5n3jota-media/videos/eval/3_eval_3_3d085060cf5254eaf2ca.mp4 +2025-01-01 11:55:54,739 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:55,751 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:56,335 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:55:56,744 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:57,754 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:58,231 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:58,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:55:58,697 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:55:58,748 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:59,746 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:00,336 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:56:00,745 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:01,749 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:02,337 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:56:02,748 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:03,708 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:03,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:04,756 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:05,753 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:06,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:07,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:08,709 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:08,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:09,339 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:56:09,749 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:10,072 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:56:10,072 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:56:10,073 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:56:10,339 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:56:10,339 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:56:10,758 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:11,756 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:12,340 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:56:12,756 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:13,362 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:56:13,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:56:13,692 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:56:13,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:13,767 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:14,341 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:56:14,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:15,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:16,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:17,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:18,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:19,707 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:19,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:20,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:21,751 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:22,751 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:23,751 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:24,751 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:25,707 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:25,751 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:26,751 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:27,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:28,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:56:28,692 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:56:28,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:29,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:30,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:30,822 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:31,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:32,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:33,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:34,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:35,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:36,707 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:36,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:37,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:38,753 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:39,753 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:40,754 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:41,754 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:42,707 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:42,755 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:43,363 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:56:43,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:56:43,692 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:56:43,755 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:44,755 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:45,756 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:46,756 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:47,756 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:47,793 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:48,765 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:48,893 DEBUG SenderThread:1799171 [sender.py:send():379] send: files +2025-01-01 11:56:48,893 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_faf38f4497652bfc9799.mp4 with policy now +2025-01-01 11:56:48,899 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:56:48,899 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:56:48,899 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:56:48,900 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:56:49,154 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/7vmr9mln-media/videos/eval/4_eval_4_faf38f4497652bfc9799.mp4 +2025-01-01 11:56:49,353 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:56:49,353 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/4_eval_4_faf38f4497652bfc9799.mp4 +2025-01-01 11:56:49,353 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval +2025-01-01 11:56:49,629 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/oq5gk9ym-media/videos/eval/4_eval_4_faf38f4497652bfc9799.mp4 +2025-01-01 11:56:49,764 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:50,353 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:56:50,772 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:51,771 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:52,771 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:52,919 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:53,778 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:54,354 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:56:54,775 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:55,778 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:56,781 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:57,794 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:58,356 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:56:58,700 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:56:58,701 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:56:58,780 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:58,852 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:59,783 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:00,784 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:01,792 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:02,791 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:03,140 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:57:03,141 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:03,141 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:03,177 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:57:03,178 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:57:03,357 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:57:03,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:57:03,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:57:03,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media +2025-01-01 11:57:03,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table/eval +2025-01-01 11:57:03,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table +2025-01-01 11:57:03,373 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:57:03,562 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/1ujmz7dp-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:57:03,606 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/7ulyk1ae-events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:57:03,656 INFO wandb-upload_2:1799171 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmptvavq6qs +2025-01-01 11:57:03,761 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:57:03,772 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:57:03,783 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:04,154 INFO SenderThread:1799171 [sender.py:send_request_log_artifact():1525] logged artifact run-iz44h5u3-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDcwNzg2Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0NDk5NQ==', 'latestArtifact': None}} +2025-01-01 11:57:04,154 DEBUG SenderThread:1799171 [sender.py:send():379] send: files +2025-01-01 11:57:04,154 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-01 11:57:04,155 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:04,155 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:57:04,155 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,156 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,156 DEBUG SenderThread:1799171 [sender.py:send():379] send: telemetry +2025-01-01 11:57:04,156 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,160 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,160 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,160 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,160 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,160 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,161 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,161 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,161 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,161 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,162 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,162 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,162 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,162 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,162 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,163 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,163 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,163 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,163 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,163 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,164 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,164 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,164 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:57:04,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:57:04,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:57:04,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:57:04,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json +2025-01-01 11:57:04,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table/eval +2025-01-01 11:57:04,389 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/miye8fbw-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:57:04,562 INFO wandb-upload_2:1799171 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpvszeb7kl +2025-01-01 11:57:04,570 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/dquj3hmf-media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json +2025-01-01 11:57:04,783 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:04,937 INFO SenderThread:1799171 [sender.py:send_request_log_artifact():1525] logged artifact run-iz44h5u3-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDcwNzg4MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0NDk5Nw==', 'latestArtifact': None}} +2025-01-01 11:57:04,937 DEBUG SenderThread:1799171 [sender.py:send():379] send: files +2025-01-01 11:57:04,937 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json with policy now +2025-01-01 11:57:04,937 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:57:04,938 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,938 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:05,174 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/bzyujr0a-media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json +2025-01-01 11:57:05,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:57:05,783 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:06,359 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:57:06,783 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:07,784 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:08,784 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:09,709 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:09,784 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:10,784 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:11,786 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:12,786 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:13,363 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:57:13,694 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:57:13,694 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:57:13,786 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:14,786 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:14,796 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:15,786 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:16,786 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:17,786 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:18,787 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:19,787 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:20,710 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:20,787 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:21,787 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:22,787 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:23,787 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:24,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:25,787 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:26,710 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:26,787 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:27,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:28,694 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:57:28,694 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:57:28,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:29,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:30,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:31,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:31,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:32,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:33,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:34,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:35,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:36,789 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:37,713 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:37,789 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:38,372 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/config.yaml +2025-01-01 11:57:38,529 DEBUG SenderThread:1799171 [sender.py:send():379] send: exit +2025-01-01 11:57:38,530 INFO SenderThread:1799171 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 11:57:38,530 INFO SenderThread:1799171 [sender.py:send_exit():588] handling runtime: 325 +2025-01-01 11:57:38,531 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:38,531 INFO SenderThread:1799171 [sender.py:send_exit():594] send defer +2025-01-01 11:57:38,531 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:38,531 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 11:57:38,531 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:38,531 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 11:57:38,531 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 1 +2025-01-01 11:57:38,531 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:38,531 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 11:57:38,532 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:38,532 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 11:57:38,532 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 2 +2025-01-01 11:57:38,532 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:38,532 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 11:57:38,532 INFO HandlerThread:1799171 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 11:57:38,532 DEBUG SystemMonitor:1799171 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 11:57:38,532 DEBUG SystemMonitor:1799171 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 11:57:38,533 INFO HandlerThread:1799171 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 11:57:38,533 INFO HandlerThread:1799171 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 11:57:38,584 INFO HandlerThread:1799171 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 11:57:38,584 INFO HandlerThread:1799171 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 11:57:38,584 INFO HandlerThread:1799171 [interfaces.py:finish():200] Joined network monitor +2025-01-01 11:57:38,584 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:38,585 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 11:57:38,585 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 3 +2025-01-01 11:57:38,585 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:57:38,585 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:38,585 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 11:57:38,585 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:38,585 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 11:57:38,585 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 4 +2025-01-01 11:57:38,585 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:38,585 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 11:57:39,372 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:57:40,373 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:57:45,148 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:57:45,149 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:45,149 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 11:57:45,149 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:45,149 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 5 +2025-01-01 11:57:45,149 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 11:57:45,149 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:57:45,150 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:45,151 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:57:45,151 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 11:57:45,151 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:45,152 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:45,152 DEBUG SenderThread:1799171 [sender.py:send():379] send: summary +2025-01-01 11:57:45,156 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:45,156 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:45,156 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 11:57:45,156 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 6 +2025-01-01 11:57:45,156 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:45,156 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 11:57:45,157 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:45,157 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 11:57:45,157 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 7 +2025-01-01 11:57:45,157 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:45,157 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:45,157 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 11:57:45,157 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:45,157 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 11:57:45,375 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:57:45,531 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:57:47,942 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 8 +2025-01-01 11:57:47,942 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:57:47,942 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:47,942 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 11:57:47,943 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:47,943 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 11:57:47,943 INFO SenderThread:1799171 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 11:57:47,945 INFO SenderThread:1799171 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 11:57:47,947 INFO SenderThread:1799171 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 11:57:47,961 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 9 +2025-01-01 11:57:47,961 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:47,961 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 11:57:47,962 DEBUG SenderThread:1799171 [sender.py:send():379] send: artifact +2025-01-01 11:57:48,145 INFO SenderThread:1799171 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjk2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 11:57:48,145 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:48,145 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 11:57:48,145 INFO SenderThread:1799171 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 11:57:48,376 INFO SenderThread:1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:57:48,376 INFO SenderThread:1799171 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files +2025-01-01 11:57:48,376 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/config.yaml config.yaml +2025-01-01 11:57:48,376 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json wandb-summary.json +2025-01-01 11:57:48,376 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:57:48,377 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/requirements.txt requirements.txt +2025-01-01 11:57:48,379 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-metadata.json wandb-metadata.json +2025-01-01 11:57:48,380 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log output.log +2025-01-01 11:57:48,380 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/diff.patch diff.patch +2025-01-01 11:57:48,380 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 11:57:48,380 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/0_eval_0_8e9e62ad5c72d29c1b12.mp4 media/videos/eval/0_eval_0_8e9e62ad5c72d29c1b12.mp4 +2025-01-01 11:57:48,382 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/1_eval_1_9e1d3d257c24ab6b4734.mp4 media/videos/eval/1_eval_1_9e1d3d257c24ab6b4734.mp4 +2025-01-01 11:57:48,382 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/2_eval_2_adc4ed4f260e51275bb5.mp4 media/videos/eval/2_eval_2_adc4ed4f260e51275bb5.mp4 +2025-01-01 11:57:48,382 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/4_eval_4_faf38f4497652bfc9799.mp4 media/videos/eval/4_eval_4_faf38f4497652bfc9799.mp4 +2025-01-01 11:57:48,382 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/3_eval_3_3d085060cf5254eaf2ca.mp4 media/videos/eval/3_eval_3_3d085060cf5254eaf2ca.mp4 +2025-01-01 11:57:48,384 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json +2025-01-01 11:57:48,384 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:57:48,384 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 10 +2025-01-01 11:57:48,384 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:48,384 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 11:57:48,386 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:48,386 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 11:57:48,386 INFO SenderThread:1799171 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:57:48,532 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:57:48,532 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:57:48,560 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/config.yaml +2025-01-01 11:57:48,597 INFO wandb-upload_2:1799171 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:57:48,598 INFO wandb-upload_3:1799171 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/requirements.txt +2025-01-01 11:57:48,624 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/ja2sr776-events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:57:48,662 INFO wandb-upload_4:1799171 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:57:48,862 INFO Thread-11 (_thread_body):1799171 [sender.py:transition_state():614] send defer: 11 +2025-01-01 11:57:48,862 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:48,862 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 11:57:48,863 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:48,863 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 11:57:48,863 INFO SenderThread:1799171 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 11:57:48,864 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 12 +2025-01-01 11:57:48,864 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:48,864 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 11:57:48,864 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:48,864 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 11:57:48,864 INFO SenderThread:1799171 [file_stream.py:finish():601] file stream finish called +2025-01-01 11:57:49,055 INFO SenderThread:1799171 [file_stream.py:finish():605] file stream finish is done +2025-01-01 11:57:49,055 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 13 +2025-01-01 11:57:49,055 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:49,055 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 11:57:49,056 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:49,056 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 11:57:49,056 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 14 +2025-01-01 11:57:49,056 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:49,056 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 11:57:49,056 DEBUG SenderThread:1799171 [sender.py:send():379] send: final +2025-01-01 11:57:49,056 DEBUG SenderThread:1799171 [sender.py:send():379] send: footer +2025-01-01 11:57:49,056 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:49,056 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 11:57:49,057 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:57:49,057 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:57:49,057 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 11:57:49,057 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: server_info +2025-01-01 11:57:49,059 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:57:49,059 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 11:57:49,059 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 11:57:49,060 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:49,110 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:57:49,110 INFO MainThread:1799171 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 11:57:49,111 INFO MainThread:1799171 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 11:57:49,111 INFO MainThread:1799171 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 11:57:49,111 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 11:57:49,111 INFO HandlerThread:1799171 [handler.py:finish():882] shutting down handler +2025-01-01 11:57:50,059 INFO WriterThread:1799171 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/run-iz44h5u3.wandb +2025-01-01 11:57:50,110 INFO SenderThread:1799171 [sender.py:finish():1615] shutting down sender +2025-01-01 11:57:50,110 INFO SenderThread:1799171 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:57:50,110 INFO SenderThread:1799171 [file_pusher.py:join():175] waiting for file pusher diff --git a/iz44h5u3/wandb/debug.log b/iz44h5u3/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..79977f5cd492bab52613a56dd4361316ee1c1c53 --- /dev/null +++ b/iz44h5u3/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-01 11:52:12,995 INFO MainThread:1798991 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 11:52:12,995 INFO MainThread:1798991 [wandb_setup.py:_flush():76] Configure stats pid to 1798991 +2025-01-01 11:52:12,995 INFO MainThread:1798991 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 11:52:12,995 INFO MainThread:1798991 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/settings +2025-01-01 11:52:12,995 INFO MainThread:1798991 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 11:52:12,995 INFO MainThread:1798991 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 11:52:12,995 INFO MainThread:1798991 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 11:52:12,995 INFO MainThread:1798991 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 11:52:12,996 INFO MainThread:1798991 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/logs/debug.log +2025-01-01 11:52:12,996 INFO MainThread:1798991 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/logs/debug-internal.log +2025-01-01 11:52:12,996 INFO MainThread:1798991 [wandb_init.py:init():569] calling init triggers +2025-01-01 11:52:12,996 INFO MainThread:1798991 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3'} +2025-01-01 11:52:12,997 INFO MainThread:1798991 [wandb_init.py:init():619] starting backend +2025-01-01 11:52:12,997 INFO MainThread:1798991 [wandb_init.py:init():623] setting up manager +2025-01-01 11:52:13,001 INFO MainThread:1798991 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 11:52:13,003 INFO MainThread:1798991 [wandb_init.py:init():631] backend started and connected +2025-01-01 11:52:13,014 INFO MainThread:1798991 [wandb_init.py:init():720] updated telemetry +2025-01-01 11:52:13,021 INFO MainThread:1798991 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 11:52:13,278 INFO MainThread:1798991 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 11:52:13,346 INFO MainThread:1798991 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 11:52:13,346 INFO MainThread:1798991 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 11:52:13,689 INFO MainThread:1798991 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 11:52:13,689 INFO MainThread:1798991 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 11:52:13,689 INFO MainThread:1798991 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 11:52:13,689 INFO MainThread:1798991 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 11:52:13,690 INFO MainThread:1798991 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 11:52:13,691 INFO MainThread:1798991 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/tb, True +2025-01-01 11:52:17,628 INFO MainThread:1798991 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-01 11:57:50,112 WARNING MsgRouterThr:1798991 [router.py:message_loop():77] message_loop has been closed diff --git a/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/code/FISH/eval_robot.py b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/config.yaml b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2a052a72ddf6ec068b71ef4cd6c737de7cf55d7 --- /dev/null +++ b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1006_0' +wandb_notes: + desc: null + value: 1006_0_req_1352_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735750333 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lady +left_book_slot: + desc: null + value: twodim diff --git a/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/diff.patch b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..625cf877496560323ca22cd6b0a075ba511a2dc6 --- /dev/null +++ b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/diff.patch @@ -0,0 +1,106 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..01bdd86 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1007_0' ++# wandb_run_id: '1002_0' + ++# all books, 6/20 demos per book ++# RGBD+mask+act history ++# wandb_run_id: '1007_0' ++ ++# RGBD+mask+contact(all ftrs)+act history ++wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..50493fe 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -490,7 +490,8 @@ class Workspace: + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] +- total_num_episodes += filtered_run.summary_metrics['episode'] ++ # total_num_episodes += filtered_run.summary_metrics['episode'] ++ total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes +@@ -513,7 +514,6 @@ class Workspace: + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + +- + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index e3031af..6f97e26 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,11 +330,11 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.01 +- self.move_to_rot_tolerance = 2.0 ++ self.move_to_trans_tolerance = 0.005 ++ self.move_to_rot_tolerance = 1.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) +- self.move_to_target_timeout = 15.0 ++ self.move_to_target_timeout = 5.0 + self.move_to_target_max_rotation = 25.0 + # only can use one of these at a time + # assert use_roll + use_pitch + use_yaw <= 1, "Can only use one of roll, pitch, yaw" diff --git a/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json new file mode 100644 index 0000000000000000000000000000000000000000..050f75761d7b6273f3088165b89fcba85be80dd0 --- /dev/null +++ b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[30, 55, 0.5454545454545454, 0.386904776118963, 0.6963961869085433, 0.4168242216276581, 0.6795179396508717, 0.4371866434938173, 0.6503013074097438, 0.45355755091642347, 0.6361154354836506]]} \ No newline at end of file diff --git a/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/0_eval_0_8e9e62ad5c72d29c1b12.mp4 b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/0_eval_0_8e9e62ad5c72d29c1b12.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2d52f7c887dc97ca5e162811f9d0dc2a47dff1fe --- /dev/null +++ b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/0_eval_0_8e9e62ad5c72d29c1b12.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e9e62ad5c72d29c1b12983b1c5060a80faaef19213c585671a0d6e714502baa +size 1474906 diff --git a/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/1_eval_1_9e1d3d257c24ab6b4734.mp4 b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/1_eval_1_9e1d3d257c24ab6b4734.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..76b03ae2aa87d6b9e7b7058bebae65f306daba4b --- /dev/null +++ b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/1_eval_1_9e1d3d257c24ab6b4734.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e1d3d257c24ab6b47342a8b2ccd131cace40a2db1f892f2fae54d5cac392913 +size 1448243 diff --git a/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/2_eval_2_adc4ed4f260e51275bb5.mp4 b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/2_eval_2_adc4ed4f260e51275bb5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..69d9e91d73e2d548635dbf37034297367eafb3aa --- /dev/null +++ b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/2_eval_2_adc4ed4f260e51275bb5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adc4ed4f260e51275bb5d919665cd0984053c6bf51e70a2f97b7ae22703a0173 +size 1444569 diff --git a/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/3_eval_3_3d085060cf5254eaf2ca.mp4 b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/3_eval_3_3d085060cf5254eaf2ca.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..98e2536654ee0b3c800fbd3e46c093e0ef6c1002 --- /dev/null +++ b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/3_eval_3_3d085060cf5254eaf2ca.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d085060cf5254eaf2ca93f0f171c3c8434e038096a28844daee2e3c33dcbc6b +size 1448998 diff --git a/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/4_eval_4_faf38f4497652bfc9799.mp4 b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/4_eval_4_faf38f4497652bfc9799.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..69b27bbaaed68b2aeed18526da7b11261481413b --- /dev/null +++ b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/4_eval_4_faf38f4497652bfc9799.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:faf38f4497652bfc9799e67edee1d3db615a4c248415463093005d199797907e +size 1436685 diff --git a/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..6529e3112dd08776d88e7411c6d4ee31974d59ab --- /dev/null +++ b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log @@ -0,0 +1,270 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735750337.629556]: resetting environment +[INFO] [1735750337.634675]: cleared current plan +[INFO] [1735750337.635528]: moving to home +[INFO] [1735750338.637203]: reached home +[INFO] [1735750338.638553]: reset action history +[INFO] [1735750341.349000]: environment reset +Starting episode 0 +[INFO] [1735750341.353011]: resetting environment +[INFO] [1735750341.357317]: cleared current plan +[INFO] [1735750341.358198]: moving to home +[INFO] [1735750342.360718]: reached home +[INFO] [1735750342.362095]: reset action history +[INFO] [1735750345.121881]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735750349.035925]: resetting environment +[INFO] [1735750349.040140]: cleared current plan +[INFO] [1735750349.040972]: moving to home +[INFO] [1735750350.043557]: reached home +[INFO] [1735750350.045305]: reset action history +[INFO] [1735750352.753612]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/episode_rosbags/episode_0_2025-01-01-11-52-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1798991.log +started roslaunch server http://158.130.50.37:34121/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1799280] +process[print_text-2]: started with pid [1799304] +process[rosbag_record-3]: started with pid [1799305] +started bagging! +[INFO] [1735750387.890210]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1735750388.610344]: Stored episode 1. +[INFO] [1735750388.610638]: Saving video... +[INFO] [1735750388.804161]: Video saved! +Starting episode 1 +[INFO] [1735750388.818306]: resetting environment +[INFO] [1735750388.832021]: cleared current plan +[INFO] [1735750388.832428]: moving to home +[INFO] [1735750392.836715]: reached home +[INFO] [1735750392.837928]: reset action history +[INFO] [1735750395.548512]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735750408.287755]: resetting environment +[INFO] [1735750408.290846]: cleared current plan +[INFO] [1735750408.291009]: moving to home +[INFO] [1735750409.292604]: reached home +[INFO] [1735750409.293043]: reset action history +[INFO] [1735750411.999972]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/episode_rosbags/episode_1_2025-01-01-11-53-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1798991.log +started roslaunch server http://158.130.50.37:36517/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1799432] +process[print_text-5]: started with pid [1799433] +process[rosbag_record-6]: started with pid [1799457] +started bagging! +For topic EE_pose: timestamp difference is 116032246 for nearest: 1735750415295200232 - target: 1735750415179167986 at idx 0 +For topic EE_pose: timestamp difference is 222821215 for nearest: 1735750415401989201 - target: 1735750415179167986 at idx 0 +For topic EE_pose: timestamp difference is 421875053 for nearest: 1735750415601043039 - target: 1735750415179167986 at idx 0 +For topic gripper_width: timestamp difference is 110434115 for nearest: 1735750427145276845 - target: 1735750427034842730 at idx 190 +For topic gripper_width: timestamp difference is 118618873 for nearest: 1735750431045394805 - target: 1735750430926775932 at idx 242 +[INFO] [1735750446.396176]: Storing episode... +[print_text-5] killing on exit +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[INFO] [1735750447.148615]: Stored episode 2. +[INFO] [1735750447.149184]: Saving video... +[WARN] [1735750447.742458]: Plan exhausted +[WARN] [1735750447.791769]: Plan exhausted +[WARN] [1735750447.822745]: Plan exhausted +[WARN] [1735750447.862735]: Plan exhausted +[INFO] [1735750447.812023]: Video saved! +Starting episode 2 +[INFO] [1735750447.864697]: resetting environment +[INFO] [1735750447.882661]: cleared current plan +[INFO] [1735750447.884843]: moving to home +[INFO] [1735750451.997038]: reached home +[INFO] [1735750451.999688]: reset action history +[INFO] [1735750454.719605]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735750465.123112]: resetting environment +[INFO] [1735750465.134193]: cleared current plan +[INFO] [1735750465.134391]: moving to home +[INFO] [1735750466.136721]: reached home +[INFO] [1735750466.137274]: reset action history +[INFO] [1735750468.846687]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/episode_rosbags/episode_2_2025-01-01-11-54-28.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1798991.log +started roslaunch server http://158.130.50.37:39639/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1799576] +process[print_text-8]: started with pid [1799577] +process[rosbag_record-9]: started with pid [1799601] +started bagging! +For topic gripper_width: timestamp difference is -162488553 for nearest: 1735750474578613904 - target: 1735750474741102457 at idx 77 +For topic gripper_width: timestamp difference is 108961583 for nearest: 1735750477178635121 - target: 1735750477069673538 at idx 109 +For topic gripper_width: timestamp difference is 101743900 for nearest: 1735750498511921608 - target: 1735750498410177708 at idx 249 +[WARN] [1735750505.022898]: Plan exhausted +[INFO] [1735750503.688006]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit[pub_text-7] killing on exit +[INFO] [1735750504.484211]: Stored episode 3. +[INFO] [1735750504.488310]: Saving video... +[INFO] [1735750504.991593]: Video saved! +Starting episode 3 +[INFO] [1735750505.038944]: resetting environment +[INFO] [1735750505.057013]: cleared current plan +[INFO] [1735750505.057781]: moving to home +[INFO] [1735750509.061089]: reached home +[INFO] [1735750509.062646]: reset action history +[INFO] [1735750511.850569]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735750514.507274]: resetting environment +[INFO] [1735750514.537836]: cleared current plan +[INFO] [1735750514.538531]: moving to home +[INFO] [1735750515.541526]: reached home +[INFO] [1735750515.543016]: reset action history +[INFO] [1735750518.268815]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/episode_rosbags/episode_3_2025-01-01-11-55-18.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1798991.log +started roslaunch server http://158.130.50.37:43727/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1799724] +process[print_text-11]: started with pid [1799725] +process[rosbag_record-12]: started with pid [1799749] +started bagging! +[INFO] [1735750553.020400]: Storing episode... +[rosbag_record-12] killing on exit[print_text-11] killing on exit[pub_text-10] killing on exit +[INFO] [1735750553.824498]: Stored episode 4. +[INFO] [1735750553.826152]: Saving video... +[INFO] [1735750554.125407]: Video saved! +Starting episode 4 +[INFO] [1735750554.196780]: resetting environment +[INFO] [1735750554.230329]: cleared current plan +[INFO] [1735750554.230578]: moving to home +[INFO] [1735750558.237759]: reached home +[INFO] [1735750558.244710]: reset action history +[INFO] [1735750561.032226]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735750569.275359]: resetting environment +[INFO] [1735750569.290373]: cleared current plan +[INFO] [1735750569.290573]: moving to home +[INFO] [1735750570.293072]: reached home +[INFO] [1735750570.298395]: reset action history +[INFO] [1735750573.077176]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/episode_rosbags/episode_4_2025-01-01-11-56-13.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1798991.log +started roslaunch server http://158.130.50.37:39995/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1799866] +process[print_text-14]: started with pid [1799867] +process[rosbag_record-15]: started with pid [1799891] +started bagging! +[INFO] [1735750607.819544]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit[pub_text-13] killing on exit +[INFO] [1735750608.579792]: Stored episode 5. +[INFO] [1735750608.581920]: Saving video... +[INFO] [1735750608.842713]: Video saved! +[INFO] [1735750608.901014]: resetting environment +[INFO] [1735750608.918232]: cleared current plan +[INFO] [1735750608.918481]: moving to home +[INFO] [1735750612.923321]: reached home +[INFO] [1735750612.926516]: reset action history +[INFO] [1735750615.725269]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/requirements.txt b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-metadata.json b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..2fca0ddae61cd81ae75283b5ce7954f6adc1083a --- /dev/null +++ b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T16:52:13.369332", + "startedAt": "2025-01-01T16:52:12.992506", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3936.501125, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.978, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.028, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 730.6149291992188 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d92b49926e832a703908bb20278a4568d45def2a --- /dev/null +++ b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "8e9e62ad5c72d29c1b12983b1c5060a80faaef19213c585671a0d6e714502baa", "size": 1474906, "path": "media/videos/eval/0_eval_0_8e9e62ad5c72d29c1b12.mp4"}, "global_step": 665, "_timestamp": 1735750623.680065, "_runtime": 290.6769058704376, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "9e1d3d257c24ab6b47342a8b2ccd131cace40a2db1f892f2fae54d5cac392913", "size": 1448243, "path": "media/videos/eval/1_eval_1_9e1d3d257c24ab6b4734.mp4"}, "eval/num_success": 5.0, "episode": 55.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "adc4ed4f260e51275bb5d919665cd0984053c6bf51e70a2f97b7ae22703a0173", "size": 1444569, "path": "media/videos/eval/2_eval_2_adc4ed4f260e51275bb5.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "3d085060cf5254eaf2ca93f0f171c3c8434e038096a28844daee2e3c33dcbc6b", "size": 1448998, "path": "media/videos/eval/3_eval_3_3d085060cf5254eaf2ca.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "faf38f4497652bfc9799e67edee1d3db615a4c248415463093005d199797907e", "size": 1436685, "path": "media/videos/eval/4_eval_4_faf38f4497652bfc9799.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://drcgdf8wijvqiwvp2mb99jsiwe4g4hld1vqlfej14dt3docmgaq8y85oqa4e5zmibhdixo32vlf596wn46e8oxlaowk3l0o0cw83tampspmfbm7jz27id2re93ht2b2l/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://3znmhp8rfhhhp7ge4jw0h3ltcmto03eprwnxh2re7bg24cq8h7zrgpx9lmyciini1hzhlncvlvuxyzev6z48a6v9bh55l5o0415pekc6du4fukmyp0ahscz593vg90sm:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 30, "total_num_episodes": 55, "total_success_rate": 0.5454545454545454, "total_success_rate_lb_0.01": 0.386904776118963, "total_success_rate_ub_0.01": 0.6963961869085433, "total_success_rate_lb_0.025": 0.4168242216276581, "total_success_rate_ub_0.025": 0.6795179396508717, "total_success_rate_lb_0.05": 0.4371866434938173, "total_success_rate_ub_0.05": 0.6503013074097438, "total_success_rate_lb_0.1": 0.45355755091642347, "total_success_rate_ub_0.1": 0.6361154354836506, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "da430633f673220357fdf073c1e35553cfb9baaeba00dbe4f8caf7a3a0de7f1c", "size": 520, "artifact_path": "wandb-client-artifact://619ch4g6aej7e07lbslesemu5l38ckfb5p83z6d9c3dembek1ugzhdqfb7sstmqw1tv0tjtr7z30xmv4b2xiq2r31no7rvoh91skn04n0d5bfikepjsjy0combicm9n0/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://kz4mickas5amof42yhm6k2z15vl085l5j2sh02wirl7ty9wktunuepn864i4aieelb80t164otun8p3mq6fz467okuan7tqi4d8zyrgarkbq7xegvq5kywxswa1sorqx:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 325}, "eval/total_success_rate": 0.5454545617103577, "eval/total_success_rate_lb_0.01": 0.386904776096344, "eval/total_success_rate_ub_0.01": 0.6963961720466614, "eval/total_success_rate_lb_0.025": 0.41682422161102295, "eval/total_success_rate_ub_0.025": 0.679517924785614, "eval/total_success_rate_lb_0.05": 0.43718665838241577, "eval/total_success_rate_ub_0.05": 0.6503012776374817, "eval/total_success_rate_lb_0.1": 0.453557550907135, "eval/total_success_rate_ub_0.1": 0.6361154317855835} \ No newline at end of file diff --git a/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/logs/debug-internal.log b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6fd48134700f66b794d0c3705439ca13eb062c3c --- /dev/null +++ b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/logs/debug-internal.log @@ -0,0 +1,846 @@ +2025-01-01 11:52:13,003 INFO StreamThr :1799171 [internal.py:wandb_internal():85] W&B internal server running at pid: 1799171, started at: 2025-01-01 11:52:13.003247 +2025-01-01 11:52:13,004 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status +2025-01-01 11:52:13,015 INFO WriterThread:1799171 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/run-iz44h5u3.wandb +2025-01-01 11:52:13,015 DEBUG SenderThread:1799171 [sender.py:send():379] send: header +2025-01-01 11:52:13,023 DEBUG SenderThread:1799171 [sender.py:send():379] send: run +2025-01-01 11:52:13,261 INFO SenderThread:1799171 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files +2025-01-01 11:52:13,262 INFO SenderThread:1799171 [sender.py:_start_run_threads():1188] run started: iz44h5u3 with start time 1735750333.003159 +2025-01-01 11:52:13,279 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 11:52:13,279 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: check_version +2025-01-01 11:52:13,351 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 11:52:13,355 DEBUG HandlerThread:1799171 [system_info.py:__init__():26] System info init +2025-01-01 11:52:13,356 DEBUG HandlerThread:1799171 [system_info.py:__init__():41] System info init done +2025-01-01 11:52:13,356 INFO HandlerThread:1799171 [system_monitor.py:start():194] Starting system monitor +2025-01-01 11:52:13,356 INFO SystemMonitor:1799171 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 11:52:13,356 INFO HandlerThread:1799171 [system_monitor.py:probe():214] Collecting system info +2025-01-01 11:52:13,356 INFO SystemMonitor:1799171 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 11:52:13,357 INFO SystemMonitor:1799171 [interfaces.py:start():188] Started disk monitoring +2025-01-01 11:52:13,357 INFO SystemMonitor:1799171 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 11:52:13,358 INFO SystemMonitor:1799171 [interfaces.py:start():188] Started memory monitoring +2025-01-01 11:52:13,358 INFO SystemMonitor:1799171 [interfaces.py:start():188] Started network monitoring +2025-01-01 11:52:13,369 DEBUG HandlerThread:1799171 [system_info.py:probe():152] Probing system +2025-01-01 11:52:13,371 DEBUG HandlerThread:1799171 [system_info.py:_probe_git():137] Probing git +2025-01-01 11:52:13,376 DEBUG HandlerThread:1799171 [system_info.py:_probe_git():145] Probing git done +2025-01-01 11:52:13,376 DEBUG HandlerThread:1799171 [system_info.py:probe():200] Probing system done +2025-01-01 11:52:13,376 DEBUG HandlerThread:1799171 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T16:52:13.369332', 'startedAt': '2025-01-01T16:52:12.992506', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3936.501125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.978, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.028, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 730.6149291992188}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 11:52:13,376 INFO HandlerThread:1799171 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 11:52:13,376 INFO HandlerThread:1799171 [system_monitor.py:probe():227] Publishing system info +2025-01-01 11:52:13,376 DEBUG HandlerThread:1799171 [system_info.py:_save_code():44] Saving code +2025-01-01 11:52:13,377 DEBUG HandlerThread:1799171 [system_info.py:_save_code():67] Saving code done +2025-01-01 11:52:13,377 DEBUG HandlerThread:1799171 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 11:52:13,440 DEBUG HandlerThread:1799171 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 11:52:13,440 INFO HandlerThread:1799171 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 11:52:13,443 DEBUG SenderThread:1799171 [sender.py:send():379] send: files +2025-01-01 11:52:13,443 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 11:52:13,444 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 11:52:13,444 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 11:52:13,688 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 11:52:13,688 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: python_packages +2025-01-01 11:52:13,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:52:13,689 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:52:13,692 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/qgxls3bd-wandb-metadata.json +2025-01-01 11:52:13,704 INFO wandb-upload_2:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/pj5o5gnf-diff.patch +2025-01-01 11:52:13,733 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:13,733 INFO HandlerThread:1799171 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/tb" + save: true +} +control { + relay_id: "11eed6c934be49448399bbc4ba5ead8a" +} +_info { + stream_id: "iz44h5u3" +} + +2025-01-01 11:52:13,733 DEBUG HandlerThread:1799171 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 11:52:13,759 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/ay0ft0ay-code/FISH/eval_robot.py +2025-01-01 11:52:13,838 DEBUG SenderThread:1799171 [sender.py:send():379] send: telemetry +2025-01-01 11:52:13,838 DEBUG SenderThread:1799171 [sender.py:send():379] send: tbrecord +2025-01-01 11:52:13,839 DEBUG SenderThread:1799171 [sender.py:send():379] send: files +2025-01-01 11:52:13,839 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 with policy live +2025-01-01 11:52:14,029 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/yn6p74yh-events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:52:14,263 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:52:14,263 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/diff.patch +2025-01-01 11:52:14,263 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/requirements.txt +2025-01-01 11:52:14,264 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/code/FISH/eval_robot.py +2025-01-01 11:52:14,264 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-metadata.json +2025-01-01 11:52:14,264 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:52:14,264 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/code +2025-01-01 11:52:14,264 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/code/FISH +2025-01-01 11:52:14,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:15,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:16,262 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:52:16,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:17,629 DEBUG SenderThread:1799171 [sender.py:send():379] send: config +2025-01-01 11:52:17,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:18,637 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:18,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:19,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:20,263 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:52:20,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:21,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:22,264 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:52:22,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:23,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:23,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:24,265 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:52:24,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:25,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:26,265 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:52:26,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:27,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:28,688 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:52:28,689 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:52:28,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:28,801 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:29,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:30,266 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:52:30,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:31,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:32,267 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:52:32,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:33,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:34,268 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:52:34,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:34,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:35,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:36,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:37,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:38,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:39,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:40,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:40,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:41,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:42,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:43,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:52:43,689 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:52:43,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:44,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:45,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:45,827 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:46,271 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/config.yaml +2025-01-01 11:52:46,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:47,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:48,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:49,693 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:50,693 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:51,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:51,693 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:52,693 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:53,693 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:54,693 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:55,694 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:56,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:52:56,694 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:57,694 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:58,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:52:58,689 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:52:58,733 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:52:59,694 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:00,694 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:01,694 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:01,829 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:02,695 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:03,695 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:04,695 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:05,695 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:06,695 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:07,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:07,695 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:08,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:08,845 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:53:08,846 DEBUG SenderThread:1799171 [sender.py:send():379] send: files +2025-01-01 11:53:08,846 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_8e9e62ad5c72d29c1b12.mp4 with policy now +2025-01-01 11:53:08,846 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:53:08,847 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:53:08,848 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:53:09,100 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/0mpf8g7o-media/videos/eval/0_eval_0_8e9e62ad5c72d29c1b12.mp4 +2025-01-01 11:53:09,277 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/0_eval_0_8e9e62ad5c72d29c1b12.mp4 +2025-01-01 11:53:09,277 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:53:09,277 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos +2025-01-01 11:53:09,277 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media +2025-01-01 11:53:09,278 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval +2025-01-01 11:53:09,546 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/cgzcydm9-media/videos/eval/0_eval_0_8e9e62ad5c72d29c1b12.mp4 +2025-01-01 11:53:09,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:10,277 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:53:10,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:11,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:12,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:12,837 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:13,359 DEBUG SystemMonitor:1799171 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 11:53:13,359 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:53:13,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:53:13,690 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:53:13,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:14,279 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:53:14,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:15,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:16,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:17,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:18,280 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:53:18,590 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:18,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:19,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:20,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:21,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:22,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:23,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:23,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:24,698 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:25,698 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:26,698 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:27,698 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:28,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:53:28,689 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:53:28,698 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:28,778 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:29,284 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:53:29,536 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/bpva2uaz-events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:53:29,698 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:30,284 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:53:30,698 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:31,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:32,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:33,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:34,285 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:53:34,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:34,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:35,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:36,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:37,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:38,287 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:53:38,700 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:39,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:39,700 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:40,700 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:41,700 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:42,700 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:43,360 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:53:43,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:53:43,690 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:53:43,737 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:44,701 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:44,799 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:45,701 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:46,701 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:47,701 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:48,290 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:53:48,701 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:49,701 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:50,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:50,701 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:51,702 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:52,291 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:53:52,702 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:53,703 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:54,703 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:55,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:53:55,703 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:56,703 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:57,703 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:58,689 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:53:58,690 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:53:58,733 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:53:59,703 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:00,704 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:00,782 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:01,704 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:02,704 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:03,704 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:04,705 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:05,705 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:06,396 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:06,705 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:07,708 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:07,858 DEBUG SenderThread:1799171 [sender.py:send():379] send: files +2025-01-01 11:54:07,858 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_9e1d3d257c24ab6b4734.mp4 with policy now +2025-01-01 11:54:07,860 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:54:07,861 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:54:07,861 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:54:07,862 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:54:08,119 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/ghelnd32-media/videos/eval/1_eval_1_9e1d3d257c24ab6b4734.mp4 +2025-01-01 11:54:08,296 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:54:08,297 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:54:08,297 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/1_eval_1_9e1d3d257c24ab6b4734.mp4 +2025-01-01 11:54:08,297 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval +2025-01-01 11:54:08,603 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/9hzuoahb-media/videos/eval/1_eval_1_9e1d3d257c24ab6b4734.mp4 +2025-01-01 11:54:08,709 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:09,711 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:10,298 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:54:10,708 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:11,708 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:11,712 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:12,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:13,360 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:54:13,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:54:13,691 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:54:13,737 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:14,299 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:54:14,711 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:15,710 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:16,300 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:54:16,711 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:16,766 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:17,711 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:18,711 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:19,711 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:20,711 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:21,712 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:22,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:22,712 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:23,712 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:24,712 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:25,304 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:54:25,712 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:26,169 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:54:26,169 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:54:26,169 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:54:26,305 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:54:26,305 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:54:26,712 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:27,712 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:28,177 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:28,306 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:54:28,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:54:28,690 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:54:28,733 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:29,713 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:30,306 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:54:30,713 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:31,713 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:32,713 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:33,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:33,713 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:34,713 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:35,713 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:36,308 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:54:36,714 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:37,714 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:38,309 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:54:38,714 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:39,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:39,714 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:40,714 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:41,714 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:42,714 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:43,361 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:54:43,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:54:43,691 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:54:43,733 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:44,715 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:44,812 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:45,715 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:46,715 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:47,715 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:48,715 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:49,715 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:50,696 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:50,715 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:51,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:52,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:53,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:54,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:55,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:54:55,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:56,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:57,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:58,690 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:54:58,691 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:54:58,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:54:59,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:00,316 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:55:00,717 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:00,837 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:01,716 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:02,717 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:03,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:04,717 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:05,037 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:55:05,037 DEBUG SenderThread:1799171 [sender.py:send():379] send: files +2025-01-01 11:55:05,038 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_adc4ed4f260e51275bb5.mp4 with policy now +2025-01-01 11:55:05,038 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:55:05,038 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:55:05,041 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:55:05,302 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/mnsqe9em-media/videos/eval/2_eval_2_adc4ed4f260e51275bb5.mp4 +2025-01-01 11:55:05,318 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:55:05,318 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/2_eval_2_adc4ed4f260e51275bb5.mp4 +2025-01-01 11:55:05,318 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval +2025-01-01 11:55:05,582 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/v90w54po-media/videos/eval/2_eval_2_adc4ed4f260e51275bb5.mp4 +2025-01-01 11:55:05,723 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:06,058 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:06,318 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:55:06,722 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:07,723 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:08,723 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:09,722 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:10,319 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:55:10,725 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:11,105 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:11,725 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:12,726 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:13,361 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:55:13,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:55:13,691 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:55:13,733 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:14,321 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:55:14,728 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:15,004 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:55:15,004 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:55:15,004 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:55:15,321 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:55:15,321 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:55:15,606 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/murc48he-events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:55:15,727 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:16,321 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:55:16,585 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:16,731 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:17,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:18,322 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:55:18,726 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:19,726 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:20,323 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:55:20,727 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:21,698 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:21,727 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:22,727 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:23,727 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:24,727 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:25,727 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:26,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:26,727 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:27,727 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:28,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:55:28,691 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:55:28,733 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:29,728 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:30,728 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:31,728 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:31,839 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:32,728 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:33,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:34,728 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:35,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:36,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:37,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:37,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:38,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:39,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:40,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:41,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:42,699 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:42,729 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:43,361 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:55:43,691 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:55:43,691 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:55:43,733 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:44,730 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:45,730 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:46,730 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:47,730 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:47,779 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:48,730 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:49,730 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:50,730 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:51,731 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:52,731 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:53,020 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:53,735 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:54,181 DEBUG SenderThread:1799171 [sender.py:send():379] send: files +2025-01-01 11:55:54,181 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_3d085060cf5254eaf2ca.mp4 with policy now +2025-01-01 11:55:54,192 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:55:54,192 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:55:54,193 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:55:54,201 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:55:54,334 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:55:54,334 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:55:54,334 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/3_eval_3_3d085060cf5254eaf2ca.mp4 +2025-01-01 11:55:54,334 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval +2025-01-01 11:55:54,435 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/71mdrzum-media/videos/eval/3_eval_3_3d085060cf5254eaf2ca.mp4 +2025-01-01 11:55:54,696 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/v5n3jota-media/videos/eval/3_eval_3_3d085060cf5254eaf2ca.mp4 +2025-01-01 11:55:54,739 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:55,751 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:56,335 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:55:56,744 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:57,754 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:58,231 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:55:58,697 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:55:58,697 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:55:58,748 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:55:59,746 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:00,336 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:56:00,745 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:01,749 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:02,337 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:56:02,748 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:03,708 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:03,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:04,756 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:05,753 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:06,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:07,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:08,709 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:08,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:09,339 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:56:09,749 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:10,072 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:56:10,072 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:56:10,073 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:56:10,339 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:56:10,339 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:56:10,758 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:11,756 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:12,340 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:56:12,756 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:13,362 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:56:13,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:56:13,692 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:56:13,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:13,767 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:14,341 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:56:14,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:15,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:16,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:17,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:18,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:19,707 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:19,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:20,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:21,751 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:22,751 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:23,751 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:24,751 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:25,707 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:25,751 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:26,751 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:27,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:28,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:56:28,692 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:56:28,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:29,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:30,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:30,822 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:31,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:32,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:33,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:34,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:35,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:36,707 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:36,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:37,752 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:38,753 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:39,753 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:40,754 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:41,754 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:42,707 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:42,755 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:43,363 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:56:43,692 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:56:43,692 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:56:43,755 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:44,755 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:45,756 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:46,756 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:47,756 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:47,793 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:48,765 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:48,893 DEBUG SenderThread:1799171 [sender.py:send():379] send: files +2025-01-01 11:56:48,893 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_faf38f4497652bfc9799.mp4 with policy now +2025-01-01 11:56:48,899 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:56:48,899 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:56:48,899 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:56:48,900 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:56:49,154 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/7vmr9mln-media/videos/eval/4_eval_4_faf38f4497652bfc9799.mp4 +2025-01-01 11:56:49,353 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:56:49,353 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/4_eval_4_faf38f4497652bfc9799.mp4 +2025-01-01 11:56:49,353 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval +2025-01-01 11:56:49,629 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/oq5gk9ym-media/videos/eval/4_eval_4_faf38f4497652bfc9799.mp4 +2025-01-01 11:56:49,764 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:50,353 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:56:50,772 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:51,771 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:52,771 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:52,919 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:53,778 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:54,354 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:56:54,775 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:55,778 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:56,781 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:57,794 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:58,356 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:56:58,700 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:56:58,701 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:56:58,780 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:56:58,852 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:56:59,783 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:00,784 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:01,792 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:02,791 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:03,140 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:57:03,141 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:03,141 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:03,177 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:57:03,178 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:57:03,357 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:57:03,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:57:03,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:57:03,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media +2025-01-01 11:57:03,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table/eval +2025-01-01 11:57:03,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table +2025-01-01 11:57:03,373 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:57:03,562 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/1ujmz7dp-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:57:03,606 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/7ulyk1ae-events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:57:03,656 INFO wandb-upload_2:1799171 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmptvavq6qs +2025-01-01 11:57:03,761 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:57:03,772 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:57:03,783 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:04,154 INFO SenderThread:1799171 [sender.py:send_request_log_artifact():1525] logged artifact run-iz44h5u3-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDcwNzg2Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0NDk5NQ==', 'latestArtifact': None}} +2025-01-01 11:57:04,154 DEBUG SenderThread:1799171 [sender.py:send():379] send: files +2025-01-01 11:57:04,154 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-01 11:57:04,155 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:04,155 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:57:04,155 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,156 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,156 DEBUG SenderThread:1799171 [sender.py:send():379] send: telemetry +2025-01-01 11:57:04,156 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,160 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,160 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,160 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,160 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,160 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,161 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,161 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,161 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,161 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,162 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,162 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,162 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,162 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,162 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,163 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,163 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,163 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,163 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,163 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,164 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,164 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:04,164 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:57:04,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:57:04,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:57:04,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:57:04,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json +2025-01-01 11:57:04,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table/eval +2025-01-01 11:57:04,389 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/miye8fbw-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:57:04,562 INFO wandb-upload_2:1799171 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpvszeb7kl +2025-01-01 11:57:04,570 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/dquj3hmf-media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json +2025-01-01 11:57:04,783 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:04,937 INFO SenderThread:1799171 [sender.py:send_request_log_artifact():1525] logged artifact run-iz44h5u3-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDcwNzg4MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0NDk5Nw==', 'latestArtifact': None}} +2025-01-01 11:57:04,937 DEBUG SenderThread:1799171 [sender.py:send():379] send: files +2025-01-01 11:57:04,937 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json with policy now +2025-01-01 11:57:04,937 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:57:04,938 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:04,938 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:05,174 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/bzyujr0a-media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json +2025-01-01 11:57:05,358 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:57:05,783 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:06,359 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:57:06,783 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:07,784 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:08,784 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:09,709 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:09,784 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:10,784 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:11,786 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:12,786 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:13,363 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:57:13,694 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:57:13,694 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:57:13,786 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:14,786 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:14,796 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:15,786 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:16,786 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:17,786 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:18,787 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:19,787 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:20,710 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:20,787 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:21,787 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:22,787 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:23,787 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:24,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:25,787 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:26,710 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:26,787 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:27,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:28,694 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:57:28,694 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:57:28,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:29,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:30,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:31,750 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:31,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:32,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:33,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:34,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:35,788 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:36,789 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:37,713 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:37,789 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:38,372 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/config.yaml +2025-01-01 11:57:38,529 DEBUG SenderThread:1799171 [sender.py:send():379] send: exit +2025-01-01 11:57:38,530 INFO SenderThread:1799171 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 11:57:38,530 INFO SenderThread:1799171 [sender.py:send_exit():588] handling runtime: 325 +2025-01-01 11:57:38,531 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:38,531 INFO SenderThread:1799171 [sender.py:send_exit():594] send defer +2025-01-01 11:57:38,531 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:38,531 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 11:57:38,531 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:38,531 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 11:57:38,531 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 1 +2025-01-01 11:57:38,531 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:38,531 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 11:57:38,532 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:38,532 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 11:57:38,532 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 2 +2025-01-01 11:57:38,532 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:38,532 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 11:57:38,532 INFO HandlerThread:1799171 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 11:57:38,532 DEBUG SystemMonitor:1799171 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 11:57:38,532 DEBUG SystemMonitor:1799171 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 11:57:38,533 INFO HandlerThread:1799171 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 11:57:38,533 INFO HandlerThread:1799171 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 11:57:38,584 INFO HandlerThread:1799171 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 11:57:38,584 INFO HandlerThread:1799171 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 11:57:38,584 INFO HandlerThread:1799171 [interfaces.py:finish():200] Joined network monitor +2025-01-01 11:57:38,584 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:38,585 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 11:57:38,585 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 3 +2025-01-01 11:57:38,585 DEBUG SenderThread:1799171 [sender.py:send():379] send: stats +2025-01-01 11:57:38,585 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:38,585 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 11:57:38,585 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:38,585 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 11:57:38,585 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 4 +2025-01-01 11:57:38,585 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:38,585 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 11:57:39,372 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:57:40,373 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:57:45,148 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:57:45,149 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:45,149 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 11:57:45,149 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:45,149 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 5 +2025-01-01 11:57:45,149 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 11:57:45,149 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:57:45,150 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:45,151 DEBUG SenderThread:1799171 [sender.py:send():379] send: history +2025-01-01 11:57:45,151 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 11:57:45,151 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:57:45,152 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:45,152 DEBUG SenderThread:1799171 [sender.py:send():379] send: summary +2025-01-01 11:57:45,156 INFO SenderThread:1799171 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:57:45,156 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:45,156 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 11:57:45,156 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 6 +2025-01-01 11:57:45,156 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:45,156 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 11:57:45,157 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:45,157 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 11:57:45,157 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 7 +2025-01-01 11:57:45,157 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:57:45,157 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:45,157 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 11:57:45,157 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:45,157 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 11:57:45,375 INFO Thread-12 :1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:57:45,531 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:57:47,942 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 8 +2025-01-01 11:57:47,942 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:57:47,942 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:47,942 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 11:57:47,943 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:47,943 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 11:57:47,943 INFO SenderThread:1799171 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 11:57:47,945 INFO SenderThread:1799171 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 11:57:47,947 INFO SenderThread:1799171 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 11:57:47,961 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 9 +2025-01-01 11:57:47,961 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:47,961 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 11:57:47,962 DEBUG SenderThread:1799171 [sender.py:send():379] send: artifact +2025-01-01 11:57:48,145 INFO SenderThread:1799171 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjk2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 11:57:48,145 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:48,145 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 11:57:48,145 INFO SenderThread:1799171 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 11:57:48,376 INFO SenderThread:1799171 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:57:48,376 INFO SenderThread:1799171 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files +2025-01-01 11:57:48,376 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/config.yaml config.yaml +2025-01-01 11:57:48,376 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json wandb-summary.json +2025-01-01 11:57:48,376 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:57:48,377 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/requirements.txt requirements.txt +2025-01-01 11:57:48,379 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-metadata.json wandb-metadata.json +2025-01-01 11:57:48,380 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log output.log +2025-01-01 11:57:48,380 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/diff.patch diff.patch +2025-01-01 11:57:48,380 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 11:57:48,380 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/0_eval_0_8e9e62ad5c72d29c1b12.mp4 media/videos/eval/0_eval_0_8e9e62ad5c72d29c1b12.mp4 +2025-01-01 11:57:48,382 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/1_eval_1_9e1d3d257c24ab6b4734.mp4 media/videos/eval/1_eval_1_9e1d3d257c24ab6b4734.mp4 +2025-01-01 11:57:48,382 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/2_eval_2_adc4ed4f260e51275bb5.mp4 media/videos/eval/2_eval_2_adc4ed4f260e51275bb5.mp4 +2025-01-01 11:57:48,382 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/4_eval_4_faf38f4497652bfc9799.mp4 media/videos/eval/4_eval_4_faf38f4497652bfc9799.mp4 +2025-01-01 11:57:48,382 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/videos/eval/3_eval_3_3d085060cf5254eaf2ca.mp4 media/videos/eval/3_eval_3_3d085060cf5254eaf2ca.mp4 +2025-01-01 11:57:48,384 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json +2025-01-01 11:57:48,384 INFO SenderThread:1799171 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:57:48,384 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 10 +2025-01-01 11:57:48,384 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:48,384 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 11:57:48,386 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:48,386 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 11:57:48,386 INFO SenderThread:1799171 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:57:48,532 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:57:48,532 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:57:48,560 INFO wandb-upload_1:1799171 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/config.yaml +2025-01-01 11:57:48,597 INFO wandb-upload_2:1799171 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/wandb-summary.json +2025-01-01 11:57:48,598 INFO wandb-upload_3:1799171 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/requirements.txt +2025-01-01 11:57:48,624 INFO wandb-upload_0:1799171 [upload_job.py:push():130] Uploaded file /tmp/tmpk59963dzwandb/ja2sr776-events.out.tfevents.1735750333.leonmkim-ROG-Strix-G15CS-G15CS.1798991.0 +2025-01-01 11:57:48,662 INFO wandb-upload_4:1799171 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/files/output.log +2025-01-01 11:57:48,862 INFO Thread-11 (_thread_body):1799171 [sender.py:transition_state():614] send defer: 11 +2025-01-01 11:57:48,862 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:48,862 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 11:57:48,863 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:48,863 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 11:57:48,863 INFO SenderThread:1799171 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 11:57:48,864 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 12 +2025-01-01 11:57:48,864 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:48,864 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 11:57:48,864 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:48,864 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 11:57:48,864 INFO SenderThread:1799171 [file_stream.py:finish():601] file stream finish called +2025-01-01 11:57:49,055 INFO SenderThread:1799171 [file_stream.py:finish():605] file stream finish is done +2025-01-01 11:57:49,055 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 13 +2025-01-01 11:57:49,055 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:49,055 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 11:57:49,056 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:49,056 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 11:57:49,056 INFO SenderThread:1799171 [sender.py:transition_state():614] send defer: 14 +2025-01-01 11:57:49,056 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:57:49,056 INFO HandlerThread:1799171 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 11:57:49,056 DEBUG SenderThread:1799171 [sender.py:send():379] send: final +2025-01-01 11:57:49,056 DEBUG SenderThread:1799171 [sender.py:send():379] send: footer +2025-01-01 11:57:49,056 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: defer +2025-01-01 11:57:49,056 INFO SenderThread:1799171 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 11:57:49,057 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:57:49,057 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:57:49,057 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 11:57:49,057 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: server_info +2025-01-01 11:57:49,059 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:57:49,059 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 11:57:49,059 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 11:57:49,060 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:57:49,110 DEBUG SenderThread:1799171 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:57:49,110 INFO MainThread:1799171 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 11:57:49,111 INFO MainThread:1799171 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 11:57:49,111 INFO MainThread:1799171 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 11:57:49,111 DEBUG HandlerThread:1799171 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 11:57:49,111 INFO HandlerThread:1799171 [handler.py:finish():882] shutting down handler +2025-01-01 11:57:50,059 INFO WriterThread:1799171 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/run-iz44h5u3.wandb +2025-01-01 11:57:50,110 INFO SenderThread:1799171 [sender.py:finish():1615] shutting down sender +2025-01-01 11:57:50,110 INFO SenderThread:1799171 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:57:50,110 INFO SenderThread:1799171 [file_pusher.py:join():175] waiting for file pusher diff --git a/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/logs/debug.log b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..79977f5cd492bab52613a56dd4361316ee1c1c53 --- /dev/null +++ b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-01 11:52:12,995 INFO MainThread:1798991 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 11:52:12,995 INFO MainThread:1798991 [wandb_setup.py:_flush():76] Configure stats pid to 1798991 +2025-01-01 11:52:12,995 INFO MainThread:1798991 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 11:52:12,995 INFO MainThread:1798991 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/settings +2025-01-01 11:52:12,995 INFO MainThread:1798991 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 11:52:12,995 INFO MainThread:1798991 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 11:52:12,995 INFO MainThread:1798991 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 11:52:12,995 INFO MainThread:1798991 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 11:52:12,996 INFO MainThread:1798991 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/logs/debug.log +2025-01-01 11:52:12,996 INFO MainThread:1798991 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/logs/debug-internal.log +2025-01-01 11:52:12,996 INFO MainThread:1798991 [wandb_init.py:init():569] calling init triggers +2025-01-01 11:52:12,996 INFO MainThread:1798991 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3'} +2025-01-01 11:52:12,997 INFO MainThread:1798991 [wandb_init.py:init():619] starting backend +2025-01-01 11:52:12,997 INFO MainThread:1798991 [wandb_init.py:init():623] setting up manager +2025-01-01 11:52:13,001 INFO MainThread:1798991 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 11:52:13,003 INFO MainThread:1798991 [wandb_init.py:init():631] backend started and connected +2025-01-01 11:52:13,014 INFO MainThread:1798991 [wandb_init.py:init():720] updated telemetry +2025-01-01 11:52:13,021 INFO MainThread:1798991 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 11:52:13,278 INFO MainThread:1798991 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 11:52:13,346 INFO MainThread:1798991 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 11:52:13,346 INFO MainThread:1798991 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 11:52:13,689 INFO MainThread:1798991 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 11:52:13,689 INFO MainThread:1798991 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 11:52:13,689 INFO MainThread:1798991 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 11:52:13,689 INFO MainThread:1798991 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 11:52:13,690 INFO MainThread:1798991 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 11:52:13,691 INFO MainThread:1798991 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/iz44h5u3/tb, True +2025-01-01 11:52:17,628 INFO MainThread:1798991 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-01 11:57:50,112 WARNING MsgRouterThr:1798991 [router.py:message_loop():77] message_loop has been closed diff --git a/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/run-iz44h5u3.wandb b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/run-iz44h5u3.wandb new file mode 100644 index 0000000000000000000000000000000000000000..3a6f08a925036e54d6a279c40bec3a6bd56dce9b Binary files /dev/null and b/iz44h5u3/wandb/run-20250101_115212-iz44h5u3/run-iz44h5u3.wandb differ diff --git a/lydwde6h/.hydra/config.yaml b/lydwde6h/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05485f98b3c08bece875a460fe25a0cce7504f90 --- /dev/null +++ b/lydwde6h/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1006_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/lydwde6h/.hydra/hydra.yaml b/lydwde6h/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c16fee740e906cc358dc0b60673c6a2ed48f613 --- /dev/null +++ b/lydwde6h/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/lydwde6h/.hydra/overrides.yaml b/lydwde6h/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/lydwde6h/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/lydwde6h/episode_rosbags/aligned_depth_to_color_K.npy b/lydwde6h/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/lydwde6h/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/lydwde6h/episode_rosbags/cam_tf_world.npy b/lydwde6h/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/lydwde6h/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/lydwde6h/episode_rosbags/color_K.npy b/lydwde6h/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/lydwde6h/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/lydwde6h/episode_rosbags/depth_K.npy b/lydwde6h/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/lydwde6h/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/lydwde6h/eval_robot.log b/lydwde6h/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..35dca3d8c5e98af1b1ebd9c093eff631ca26df41 --- /dev/null +++ b/lydwde6h/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-01 11:15:58,216][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 11:15:58,219][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/lydwde6h/eval_video/0_eval.mp4 b/lydwde6h/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..292e92b727b3520b24cd1261f172cc754a0cbf1b --- /dev/null +++ b/lydwde6h/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:880040d892346c7fbdeb4b37f1632611870a673ca2eaf223b34f6e2393428aa5 +size 1440847 diff --git a/lydwde6h/eval_video/1_eval.mp4 b/lydwde6h/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..901fa41b06b1be66026883e2dfed989efd4585e5 --- /dev/null +++ b/lydwde6h/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d027f920cfefcbf4542375854542541e64d2284a4aeec23e11720b75b6aca53 +size 1220755 diff --git a/lydwde6h/eval_video/2_eval.mp4 b/lydwde6h/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..33485d00d3a7790a0dddff3f6be9956674e30d95 --- /dev/null +++ b/lydwde6h/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7105aabd534b6cd8cb8e49c2530d962bf85f4c2c06674dfa2012c7e3aed5f567 +size 873820 diff --git a/lydwde6h/eval_video/3_eval.mp4 b/lydwde6h/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0c0ba9783fb64bf2f35d64d41d25547b334c00df --- /dev/null +++ b/lydwde6h/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0346db17ff10377ae0825e93b1be0b59b49cff20bee80a503b2e42c8786c237 +size 949006 diff --git a/lydwde6h/eval_video/4_eval.mp4 b/lydwde6h/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c68fdeeba854234f5d770688a33dd9dfb9be14de --- /dev/null +++ b/lydwde6h/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77f21e81cd3b0f937629b2dc7a6931d3c6efa5c1c524b45ce54f6cdbba41c392 +size 840784 diff --git a/lydwde6h/tb/events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 b/lydwde6h/tb/events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 new file mode 100644 index 0000000000000000000000000000000000000000..c97ccb91e4fa973277bc5b106729a061035b9645 --- /dev/null +++ b/lydwde6h/tb/events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a69a73677b4ade3e0d8be64090b292b8714cfe5df9f46e496ab8f4f2ce91689c +size 6539 diff --git a/lydwde6h/wandb/debug-internal.log b/lydwde6h/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..358074d761076bdee4378040f931d18d9c7baf5c --- /dev/null +++ b/lydwde6h/wandb/debug-internal.log @@ -0,0 +1,883 @@ +2025-01-01 11:16:04,052 INFO StreamThr :1791826 [internal.py:wandb_internal():85] W&B internal server running at pid: 1791826, started at: 2025-01-01 11:16:04.052436 +2025-01-01 11:16:04,053 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status +2025-01-01 11:16:04,065 INFO WriterThread:1791826 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/run-lydwde6h.wandb +2025-01-01 11:16:04,065 DEBUG SenderThread:1791826 [sender.py:send():379] send: header +2025-01-01 11:16:04,072 DEBUG SenderThread:1791826 [sender.py:send():379] send: run +2025-01-01 11:16:04,268 INFO SenderThread:1791826 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files +2025-01-01 11:16:04,268 INFO SenderThread:1791826 [sender.py:_start_run_threads():1188] run started: lydwde6h with start time 1735748164.052899 +2025-01-01 11:16:04,285 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 11:16:04,286 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: check_version +2025-01-01 11:16:04,505 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 11:16:04,510 DEBUG HandlerThread:1791826 [system_info.py:__init__():26] System info init +2025-01-01 11:16:04,510 DEBUG HandlerThread:1791826 [system_info.py:__init__():41] System info init done +2025-01-01 11:16:04,510 INFO HandlerThread:1791826 [system_monitor.py:start():194] Starting system monitor +2025-01-01 11:16:04,510 INFO SystemMonitor:1791826 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 11:16:04,511 INFO HandlerThread:1791826 [system_monitor.py:probe():214] Collecting system info +2025-01-01 11:16:04,511 INFO SystemMonitor:1791826 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 11:16:04,512 INFO SystemMonitor:1791826 [interfaces.py:start():188] Started disk monitoring +2025-01-01 11:16:04,513 INFO SystemMonitor:1791826 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 11:16:04,513 INFO SystemMonitor:1791826 [interfaces.py:start():188] Started memory monitoring +2025-01-01 11:16:04,513 INFO SystemMonitor:1791826 [interfaces.py:start():188] Started network monitoring +2025-01-01 11:16:04,523 DEBUG HandlerThread:1791826 [system_info.py:probe():152] Probing system +2025-01-01 11:16:04,525 DEBUG HandlerThread:1791826 [system_info.py:_probe_git():137] Probing git +2025-01-01 11:16:04,529 DEBUG HandlerThread:1791826 [system_info.py:_probe_git():145] Probing git done +2025-01-01 11:16:04,529 DEBUG HandlerThread:1791826 [system_info.py:probe():200] Probing system done +2025-01-01 11:16:04,529 DEBUG HandlerThread:1791826 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T16:16:04.523898', 'startedAt': '2025-01-01T16:16:04.043345', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.45725, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.976, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 688.4198722839355}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 11:16:04,530 INFO HandlerThread:1791826 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 11:16:04,530 INFO HandlerThread:1791826 [system_monitor.py:probe():227] Publishing system info +2025-01-01 11:16:04,530 DEBUG HandlerThread:1791826 [system_info.py:_save_code():44] Saving code +2025-01-01 11:16:04,535 DEBUG HandlerThread:1791826 [system_info.py:_save_code():67] Saving code done +2025-01-01 11:16:04,535 DEBUG HandlerThread:1791826 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 11:16:04,599 DEBUG HandlerThread:1791826 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 11:16:04,599 INFO HandlerThread:1791826 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 11:16:04,602 DEBUG SenderThread:1791826 [sender.py:send():379] send: files +2025-01-01 11:16:04,602 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 11:16:04,603 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 11:16:04,603 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 11:16:04,844 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/wasgjph7-wandb-metadata.json +2025-01-01 11:16:04,849 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 11:16:04,849 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: python_packages +2025-01-01 11:16:04,850 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:16:04,850 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:16:04,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:04,852 INFO HandlerThread:1791826 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/tb" + save: true +} +control { + relay_id: "22470aca6a1243059732c3d7d33e83c8" +} +_info { + stream_id: "lydwde6h" +} + +2025-01-01 11:16:04,853 INFO wandb-upload_1:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/40lgz95o-code/FISH/eval_robot.py +2025-01-01 11:16:04,853 DEBUG HandlerThread:1791826 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 11:16:04,864 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/1n6sm2cn-diff.patch +2025-01-01 11:16:04,909 DEBUG SenderThread:1791826 [sender.py:send():379] send: telemetry +2025-01-01 11:16:04,909 DEBUG SenderThread:1791826 [sender.py:send():379] send: tbrecord +2025-01-01 11:16:04,910 DEBUG SenderThread:1791826 [sender.py:send():379] send: files +2025-01-01 11:16:04,910 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 with policy live +2025-01-01 11:16:05,095 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/nylvs93l-events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:16:05,269 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/code/FISH/eval_robot.py +2025-01-01 11:16:05,269 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:16:05,269 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/requirements.txt +2025-01-01 11:16:05,270 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/diff.patch +2025-01-01 11:16:05,270 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-metadata.json +2025-01-01 11:16:05,270 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:05,270 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/code/FISH +2025-01-01 11:16:05,270 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/code +2025-01-01 11:16:05,850 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:06,850 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:07,268 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:07,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:08,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:09,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:09,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:10,408 DEBUG SenderThread:1791826 [sender.py:send():379] send: config +2025-01-01 11:16:10,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:11,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:12,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:13,270 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:13,852 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:14,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:15,195 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:15,271 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:15,852 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:16,852 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:17,271 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:17,852 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:18,852 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:19,272 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:19,850 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:16:19,850 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:16:19,893 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:20,360 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:20,853 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:21,273 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:21,853 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:22,853 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:23,273 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:23,853 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:24,853 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:25,274 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:25,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:25,855 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:26,855 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:27,855 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:28,855 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:29,856 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:30,855 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:31,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:31,856 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:32,856 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:33,857 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:34,850 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:16:34,850 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:16:34,893 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:35,276 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:35,857 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:36,857 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:36,907 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:37,277 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/config.yaml +2025-01-01 11:16:37,857 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:38,857 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:39,278 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:39,858 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:40,857 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:41,278 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:41,858 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:42,489 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:42,858 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:43,858 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:44,860 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:45,860 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:46,860 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:47,854 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:47,860 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:48,860 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:49,850 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:16:49,850 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:16:49,897 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:50,860 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:51,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:52,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:52,968 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:53,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:54,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:55,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:56,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:57,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:58,854 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:58,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:59,837 DEBUG SenderThread:1791826 [sender.py:send():379] send: files +2025-01-01 11:16:59,838 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_880040d892346c7fbdeb.mp4 with policy now +2025-01-01 11:16:59,838 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:16:59,839 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:16:59,839 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:16:59,840 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:16:59,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:00,099 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/9gy5yukg-media/videos/eval/0_eval_0_880040d892346c7fbdeb.mp4 +2025-01-01 11:17:00,283 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/0_eval_0_880040d892346c7fbdeb.mp4 +2025-01-01 11:17:00,283 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:17:00,284 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media +2025-01-01 11:17:00,284 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval +2025-01-01 11:17:00,284 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos +2025-01-01 11:17:00,576 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/16cnv1b2-media/videos/eval/0_eval_0_880040d892346c7fbdeb.mp4 +2025-01-01 11:17:00,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:01,284 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:17:01,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:02,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:03,854 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:03,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:04,514 DEBUG SystemMonitor:1791826 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 11:17:04,514 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:17:04,850 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:17:04,851 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:17:04,897 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:05,864 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:06,864 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:07,286 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:17:07,863 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:08,863 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:08,950 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:09,864 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:10,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:11,287 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:17:11,864 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:12,864 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:13,287 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:17:13,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:14,709 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:14,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:15,288 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:17:15,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:16,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:17,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:18,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:19,290 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:17:19,556 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/3pu9iyuc-events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:17:19,850 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:17:19,851 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:17:19,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:19,995 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:20,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:21,290 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:17:21,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:22,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:23,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:24,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:25,292 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:17:25,855 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:25,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:26,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:27,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:28,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:29,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:30,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:31,141 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:31,294 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:17:31,867 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:32,867 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:33,867 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:34,515 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:17:34,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:17:34,851 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:17:34,897 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:35,295 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:17:35,867 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:36,867 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:36,989 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:37,867 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:38,867 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:39,867 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:40,868 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:41,868 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:42,855 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:42,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:43,869 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:44,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:45,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:46,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:47,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:48,855 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:48,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:49,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:17:49,851 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:17:49,893 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:50,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:51,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:52,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:53,871 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:53,912 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:54,871 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:55,871 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:56,871 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:57,871 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:58,871 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:59,855 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:59,872 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:00,871 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:01,871 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:02,876 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:02,960 DEBUG SenderThread:1791826 [sender.py:send():379] send: files +2025-01-01 11:18:02,960 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6d027f920cfefcbf4542.mp4 with policy now +2025-01-01 11:18:02,963 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:18:02,963 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:18:02,964 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:18:02,964 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:18:03,210 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/o4jnrdip-media/videos/eval/1_eval_1_6d027f920cfefcbf4542.mp4 +2025-01-01 11:18:03,304 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:18:03,304 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:03,304 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/1_eval_1_6d027f920cfefcbf4542.mp4 +2025-01-01 11:18:03,304 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval +2025-01-01 11:18:03,556 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/ao2hm548-media/videos/eval/1_eval_1_6d027f920cfefcbf4542.mp4 +2025-01-01 11:18:03,879 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:04,515 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:18:04,859 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:18:04,859 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:18:04,901 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:04,927 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:05,304 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:05,878 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:06,881 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:07,878 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:08,879 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:09,306 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:09,880 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:10,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:10,882 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:11,879 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:12,887 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:13,884 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:14,891 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:15,308 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:15,893 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:16,827 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:16,887 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:17,887 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:18,887 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:19,309 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:19,856 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:18:19,856 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:18:19,901 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:20,887 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:21,888 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:22,003 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:22,888 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:23,889 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:24,888 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:25,888 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:26,296 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:18:26,296 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:18:26,298 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:18:26,311 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:18:26,311 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:18:26,560 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/16zi9lt4-events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:18:26,888 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:27,298 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:27,311 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:27,889 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:28,888 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:29,888 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:30,890 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:31,889 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:32,753 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:32,889 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:33,314 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:33,889 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:34,516 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:18:34,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:18:34,863 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:18:34,909 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:35,890 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:36,890 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:37,315 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:37,853 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:37,890 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:38,891 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:39,316 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:39,891 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:40,891 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:41,316 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:41,891 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:42,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:42,891 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:43,891 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:44,892 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:45,892 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:46,892 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:47,892 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:48,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:48,893 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:49,857 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:18:49,857 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:18:49,901 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:50,893 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:51,893 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:52,894 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:53,894 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:53,943 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:54,894 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:55,894 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:56,895 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:57,894 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:58,894 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:59,546 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:59,899 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:00,159 DEBUG SenderThread:1791826 [sender.py:send():379] send: files +2025-01-01 11:19:00,159 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_7105aabd534b6cd8cb8e.mp4 with policy now +2025-01-01 11:19:00,160 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:19:00,161 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:19:00,161 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:19:00,163 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:19:00,323 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:19:00,323 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/2_eval_2_7105aabd534b6cd8cb8e.mp4 +2025-01-01 11:19:00,323 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval +2025-01-01 11:19:00,396 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/14weo606-media/videos/eval/2_eval_2_7105aabd534b6cd8cb8e.mp4 +2025-01-01 11:19:00,638 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/ixt8ygs3-media/videos/eval/2_eval_2_7105aabd534b6cd8cb8e.mp4 +2025-01-01 11:19:00,898 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:01,323 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:01,898 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:02,907 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:03,903 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:04,516 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:19:04,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:19:04,866 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:19:04,913 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:04,990 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:05,912 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:06,912 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:07,325 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:07,913 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:08,915 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:09,917 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:10,277 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:10,922 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:11,326 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:11,930 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:12,932 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:13,327 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:13,930 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:14,931 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:15,328 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:15,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:15,933 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:16,935 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:17,938 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:18,941 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:19,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:19:19,861 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:19:19,941 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:20,940 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:20,970 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:21,936 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:22,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:23,940 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:24,144 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:19:24,145 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:19:24,146 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:19:24,331 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:19:24,331 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:19:24,948 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:25,331 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:25,945 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:26,878 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:26,945 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:27,949 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:28,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:29,332 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:29,943 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:30,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:31,883 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:31,943 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:32,945 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:33,950 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:34,517 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:19:34,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:19:34,867 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:19:34,945 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:35,334 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:35,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:36,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:36,965 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:37,945 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:38,945 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:39,336 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:39,945 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:40,945 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:41,336 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:41,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:42,873 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:42,945 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:43,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:44,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:45,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:46,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:47,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:48,873 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:48,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:49,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:19:49,861 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:19:49,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:50,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:51,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:52,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:53,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:53,973 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:54,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:55,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:56,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:57,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:58,949 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:59,342 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:59,449 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:59,959 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:00,025 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:20:00,025 DEBUG SenderThread:1791826 [sender.py:send():379] send: files +2025-01-01 11:20:00,026 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f0346db17ff10377ae08.mp4 with policy now +2025-01-01 11:20:00,026 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:20:00,026 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:20:00,026 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:20:00,269 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/2yzey7os-media/videos/eval/3_eval_3_f0346db17ff10377ae08.mp4 +2025-01-01 11:20:00,343 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:20:00,343 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/3_eval_3_f0346db17ff10377ae08.mp4 +2025-01-01 11:20:00,343 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval +2025-01-01 11:20:00,599 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/shkrhvhq-media/videos/eval/3_eval_3_f0346db17ff10377ae08.mp4 +2025-01-01 11:20:00,961 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:01,343 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:01,958 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:02,965 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:03,963 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:04,517 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:20:04,518 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:04,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:20:04,862 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:20:04,962 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:05,976 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:06,967 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:07,345 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:07,974 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:08,980 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:09,880 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:09,967 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:10,985 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:11,346 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:11,982 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:12,987 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:13,347 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:13,988 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:14,889 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:14,991 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:15,348 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:15,989 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:16,995 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:17,991 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:18,990 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:19,211 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:20:19,211 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:20:19,211 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:20:19,349 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:20:19,349 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:20:19,349 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:19,594 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/bg9vir2h-events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:20:19,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:20:19,862 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:20:19,953 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:19,999 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:20,993 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:21,996 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:22,994 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:24,001 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:25,002 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:25,351 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:25,887 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:26,006 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:27,007 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:28,005 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:29,007 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:29,352 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:30,008 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:31,012 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:31,353 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:31,882 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:32,019 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:33,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:33,354 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:34,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:34,518 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:20:34,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:20:34,862 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:20:35,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:35,355 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:36,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:37,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:37,201 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:37,355 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:38,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:39,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:39,356 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:40,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:41,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:41,357 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:42,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:42,881 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:43,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:44,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:45,014 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:46,014 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:47,014 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:47,881 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:48,014 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:49,014 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:49,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:20:49,862 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:20:50,014 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:51,017 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:52,022 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:52,361 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/4_eval_4_77f21e81cd3b0f937629.mp4 +2025-01-01 11:20:52,361 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval +2025-01-01 11:20:52,382 DEBUG SenderThread:1791826 [sender.py:send():379] send: files +2025-01-01 11:20:52,382 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_77f21e81cd3b0f937629.mp4 with policy now +2025-01-01 11:20:52,387 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:20:52,388 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:20:52,388 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:20:52,388 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:20:52,618 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/gkncnkvn-media/videos/eval/4_eval_4_77f21e81cd3b0f937629.mp4 +2025-01-01 11:20:53,019 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:53,361 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:53,361 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/4_eval_4_77f21e81cd3b0f937629.mp4 +2025-01-01 11:20:53,361 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:20:53,424 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:53,637 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/zl7rilq9-media/videos/eval/4_eval_4_77f21e81cd3b0f937629.mp4 +2025-01-01 11:20:54,028 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:55,023 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:55,362 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:56,040 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:57,024 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:58,031 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:58,473 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:59,036 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:59,363 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:21:00,045 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:01,051 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:02,050 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:03,053 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:03,364 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:21:03,534 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:04,049 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:04,518 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:21:04,877 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:21:04,877 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:21:05,056 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:05,365 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:21:06,051 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:07,052 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:07,366 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:21:08,054 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:09,057 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:09,449 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:10,075 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:11,061 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:12,068 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:13,070 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:14,064 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:14,277 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:21:14,278 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:14,278 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:14,368 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:21:14,368 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:21:14,425 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:21:14,426 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:21:14,623 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:21:14,847 INFO wandb-upload_0:1791826 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpd19lb3vg +2025-01-01 11:21:15,019 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:21:15,028 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:21:15,056 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:15,256 INFO SenderThread:1791826 [sender.py:send_request_log_artifact():1525] logged artifact run-lydwde6h-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY4MzQxNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MTg2Ng==', 'latestArtifact': None}} +2025-01-01 11:21:15,256 DEBUG SenderThread:1791826 [sender.py:send():379] send: files +2025-01-01 11:21:15,256 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-01 11:21:15,256 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:15,256 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:21:15,256 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,257 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,257 DEBUG SenderThread:1791826 [sender.py:send():379] send: telemetry +2025-01-01 11:21:15,257 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,258 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,258 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,260 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,260 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,261 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,261 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,261 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,261 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,262 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,262 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,262 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,262 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,263 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,263 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,263 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,263 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,263 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,264 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,264 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,264 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,265 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,265 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:21:15,369 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:21:15,369 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:21:15,369 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:21:15,369 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/table/eval/total_success_rate_ci_6_a25831f94f1c283f8134.table.json +2025-01-01 11:21:15,369 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:21:15,369 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media +2025-01-01 11:21:15,370 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/table/eval +2025-01-01 11:21:15,370 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/table +2025-01-01 11:21:15,474 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/m2mqyv95-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:21:15,582 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/nducn3wp-media/table/eval/total_success_rate_ci_6_a25831f94f1c283f8134.table.json +2025-01-01 11:21:15,711 INFO wandb-upload_1:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/cf06v01f-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:21:15,870 INFO wandb-upload_2:1791826 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmexu6gcz +2025-01-01 11:21:16,056 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:16,267 INFO SenderThread:1791826 [sender.py:send_request_log_artifact():1525] logged artifact run-lydwde6h-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY4MzQyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MTg2Nw==', 'latestArtifact': None}} +2025-01-01 11:21:16,267 DEBUG SenderThread:1791826 [sender.py:send():379] send: files +2025-01-01 11:21:16,267 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_a25831f94f1c283f8134.table.json with policy now +2025-01-01 11:21:16,267 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:21:16,267 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:16,268 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:16,369 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:21:16,504 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/lquv2xxs-media/table/eval/total_success_rate_ci_6_a25831f94f1c283f8134.table.json +2025-01-01 11:21:17,056 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:17,370 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:21:18,056 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:19,057 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:19,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:21:19,862 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:21:20,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:21,004 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:21,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:22,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:23,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:24,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:25,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:26,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:26,891 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:27,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:28,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:29,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:30,059 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:31,059 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:31,894 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:32,059 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:32,375 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/config.yaml +2025-01-01 11:21:33,059 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:34,059 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:34,519 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:21:34,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:21:34,862 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:21:35,059 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:36,059 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:37,060 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:37,933 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:38,060 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:39,060 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:40,060 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:41,060 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:42,060 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:42,941 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:42,942 DEBUG SenderThread:1791826 [sender.py:send():379] send: exit +2025-01-01 11:21:42,942 INFO SenderThread:1791826 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 11:21:42,942 INFO SenderThread:1791826 [sender.py:send_exit():588] handling runtime: 338 +2025-01-01 11:21:42,942 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:42,942 INFO SenderThread:1791826 [sender.py:send_exit():594] send defer +2025-01-01 11:21:42,942 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:42,942 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 11:21:42,943 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:42,943 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 11:21:42,943 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 1 +2025-01-01 11:21:42,943 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:42,943 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 11:21:42,943 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:42,943 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 11:21:42,943 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 2 +2025-01-01 11:21:42,943 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:42,943 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 11:21:42,943 INFO HandlerThread:1791826 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 11:21:42,944 DEBUG SystemMonitor:1791826 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 11:21:42,944 DEBUG SystemMonitor:1791826 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 11:21:42,944 INFO HandlerThread:1791826 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 11:21:42,944 INFO HandlerThread:1791826 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 11:21:42,994 INFO HandlerThread:1791826 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 11:21:42,994 INFO HandlerThread:1791826 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 11:21:42,994 INFO HandlerThread:1791826 [interfaces.py:finish():200] Joined network monitor +2025-01-01 11:21:42,995 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:42,995 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 11:21:42,995 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 3 +2025-01-01 11:21:42,995 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:21:42,995 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:42,995 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 11:21:42,996 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:42,996 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 11:21:42,996 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 4 +2025-01-01 11:21:42,996 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:42,996 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 11:21:43,380 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:21:45,381 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:21:49,286 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:21:49,286 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:49,286 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 11:21:49,287 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:49,287 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 11:21:49,287 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 5 +2025-01-01 11:21:49,287 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:21:49,287 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:21:49,288 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:49,288 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:49,288 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 11:21:49,288 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:49,289 DEBUG SenderThread:1791826 [sender.py:send():379] send: summary +2025-01-01 11:21:49,292 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:49,293 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:49,293 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 11:21:49,293 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 6 +2025-01-01 11:21:49,293 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:49,293 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 11:21:49,293 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:49,293 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 11:21:49,293 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 7 +2025-01-01 11:21:49,293 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:49,293 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:49,293 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 11:21:49,294 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:49,294 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 11:21:49,382 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:21:49,943 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:21:52,947 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 8 +2025-01-01 11:21:52,947 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:21:52,948 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:52,948 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 11:21:52,948 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:52,948 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 11:21:52,948 INFO SenderThread:1791826 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 11:21:52,950 INFO SenderThread:1791826 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 11:21:52,952 INFO SenderThread:1791826 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 11:21:52,967 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 9 +2025-01-01 11:21:52,967 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:52,967 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 11:21:52,967 DEBUG SenderThread:1791826 [sender.py:send():379] send: artifact +2025-01-01 11:21:53,150 INFO SenderThread:1791826 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjk2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 11:21:53,150 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:53,150 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 11:21:53,150 INFO SenderThread:1791826 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 11:21:53,383 INFO SenderThread:1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:21:53,383 INFO SenderThread:1791826 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files +2025-01-01 11:21:53,383 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/config.yaml config.yaml +2025-01-01 11:21:53,384 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json wandb-summary.json +2025-01-01 11:21:53,384 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/requirements.txt requirements.txt +2025-01-01 11:21:53,384 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-metadata.json wandb-metadata.json +2025-01-01 11:21:53,384 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:21:53,384 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log output.log +2025-01-01 11:21:53,384 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/diff.patch diff.patch +2025-01-01 11:21:53,387 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 11:21:53,389 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/1_eval_1_6d027f920cfefcbf4542.mp4 media/videos/eval/1_eval_1_6d027f920cfefcbf4542.mp4 +2025-01-01 11:21:53,389 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/4_eval_4_77f21e81cd3b0f937629.mp4 media/videos/eval/4_eval_4_77f21e81cd3b0f937629.mp4 +2025-01-01 11:21:53,389 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/0_eval_0_880040d892346c7fbdeb.mp4 media/videos/eval/0_eval_0_880040d892346c7fbdeb.mp4 +2025-01-01 11:21:53,390 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/2_eval_2_7105aabd534b6cd8cb8e.mp4 media/videos/eval/2_eval_2_7105aabd534b6cd8cb8e.mp4 +2025-01-01 11:21:53,390 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/3_eval_3_f0346db17ff10377ae08.mp4 media/videos/eval/3_eval_3_f0346db17ff10377ae08.mp4 +2025-01-01 11:21:53,390 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/table/eval/total_success_rate_ci_6_a25831f94f1c283f8134.table.json media/table/eval/total_success_rate_ci_6_a25831f94f1c283f8134.table.json +2025-01-01 11:21:53,390 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:21:53,390 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 10 +2025-01-01 11:21:53,390 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:53,390 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 11:21:53,393 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:53,393 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 11:21:53,393 INFO SenderThread:1791826 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:21:53,569 INFO wandb-upload_1:1791826 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:21:53,571 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/config.yaml +2025-01-01 11:21:53,595 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/requirements.txt +2025-01-01 11:21:53,607 INFO wandb-upload_4:1791826 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:21:53,668 INFO wandb-upload_3:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/7sn15jvp-events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:21:53,868 INFO Thread-11 (_thread_body):1791826 [sender.py:transition_state():614] send defer: 11 +2025-01-01 11:21:53,869 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:53,869 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 11:21:53,869 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:53,869 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 11:21:53,869 INFO SenderThread:1791826 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 11:21:53,870 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 12 +2025-01-01 11:21:53,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:53,870 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 11:21:53,871 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:53,871 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 11:21:53,871 INFO SenderThread:1791826 [file_stream.py:finish():601] file stream finish called +2025-01-01 11:21:53,944 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:21:54,071 INFO SenderThread:1791826 [file_stream.py:finish():605] file stream finish is done +2025-01-01 11:21:54,071 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 13 +2025-01-01 11:21:54,071 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:21:54,071 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:54,071 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 11:21:54,071 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:54,071 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 11:21:54,072 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 14 +2025-01-01 11:21:54,072 DEBUG SenderThread:1791826 [sender.py:send():379] send: final +2025-01-01 11:21:54,072 DEBUG SenderThread:1791826 [sender.py:send():379] send: footer +2025-01-01 11:21:54,072 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:54,072 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 11:21:54,072 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:54,072 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 11:21:54,073 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:21:54,073 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:21:54,073 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 11:21:54,073 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:21:54,073 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 11:21:54,074 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: server_info +2025-01-01 11:21:54,075 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 11:21:54,076 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:54,126 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:21:54,127 INFO MainThread:1791826 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 11:21:54,127 INFO MainThread:1791826 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 11:21:54,127 INFO MainThread:1791826 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 11:21:54,128 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 11:21:54,128 INFO HandlerThread:1791826 [handler.py:finish():882] shutting down handler +2025-01-01 11:21:55,074 INFO WriterThread:1791826 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/run-lydwde6h.wandb +2025-01-01 11:21:55,127 INFO SenderThread:1791826 [sender.py:finish():1615] shutting down sender +2025-01-01 11:21:55,127 INFO SenderThread:1791826 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:21:55,127 INFO SenderThread:1791826 [file_pusher.py:join():175] waiting for file pusher diff --git a/lydwde6h/wandb/debug.log b/lydwde6h/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8509aaf710a322ab8494879adc1d2b7e39992e45 --- /dev/null +++ b/lydwde6h/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-01 11:16:04,046 INFO MainThread:1791650 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 11:16:04,046 INFO MainThread:1791650 [wandb_setup.py:_flush():76] Configure stats pid to 1791650 +2025-01-01 11:16:04,046 INFO MainThread:1791650 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/settings +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/logs/debug.log +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/logs/debug-internal.log +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_init.py:init():569] calling init triggers +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h'} +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_init.py:init():619] starting backend +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_init.py:init():623] setting up manager +2025-01-01 11:16:04,051 INFO MainThread:1791650 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 11:16:04,052 INFO MainThread:1791650 [wandb_init.py:init():631] backend started and connected +2025-01-01 11:16:04,064 INFO MainThread:1791650 [wandb_init.py:init():720] updated telemetry +2025-01-01 11:16:04,070 INFO MainThread:1791650 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 11:16:04,285 INFO MainThread:1791650 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 11:16:04,500 INFO MainThread:1791650 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 11:16:04,500 INFO MainThread:1791650 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 11:16:04,850 INFO MainThread:1791650 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 11:16:04,850 INFO MainThread:1791650 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 11:16:04,850 INFO MainThread:1791650 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 11:16:04,850 INFO MainThread:1791650 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 11:16:04,852 INFO MainThread:1791650 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 11:16:04,852 INFO MainThread:1791650 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/tb, True +2025-01-01 11:16:10,408 INFO MainThread:1791650 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-01 11:21:55,128 WARNING MsgRouterThr:1791650 [router.py:message_loop():77] message_loop has been closed diff --git a/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/code/FISH/eval_robot.py b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/config.yaml b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52f0f3a907bf6ff0bc40f1c002b9747c58bcd5bd --- /dev/null +++ b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1006_0' +wandb_notes: + desc: null + value: 1006_0_req_1352_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735748164 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/diff.patch b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..625cf877496560323ca22cd6b0a075ba511a2dc6 --- /dev/null +++ b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/diff.patch @@ -0,0 +1,106 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..01bdd86 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1007_0' ++# wandb_run_id: '1002_0' + ++# all books, 6/20 demos per book ++# RGBD+mask+act history ++# wandb_run_id: '1007_0' ++ ++# RGBD+mask+contact(all ftrs)+act history ++wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..50493fe 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -490,7 +490,8 @@ class Workspace: + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] +- total_num_episodes += filtered_run.summary_metrics['episode'] ++ # total_num_episodes += filtered_run.summary_metrics['episode'] ++ total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes +@@ -513,7 +514,6 @@ class Workspace: + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + +- + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index e3031af..6f97e26 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,11 +330,11 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.01 +- self.move_to_rot_tolerance = 2.0 ++ self.move_to_trans_tolerance = 0.005 ++ self.move_to_rot_tolerance = 1.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) +- self.move_to_target_timeout = 15.0 ++ self.move_to_target_timeout = 5.0 + self.move_to_target_max_rotation = 25.0 + # only can use one of these at a time + # assert use_roll + use_pitch + use_yaw <= 1, "Can only use one of roll, pitch, yaw" diff --git a/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/table/eval/total_success_rate_ci_6_a25831f94f1c283f8134.table.json b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/table/eval/total_success_rate_ci_6_a25831f94f1c283f8134.table.json new file mode 100644 index 0000000000000000000000000000000000000000..c45940a16e904425962406ea7b35450d0cdd450c --- /dev/null +++ b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/table/eval/total_success_rate_ci_6_a25831f94f1c283f8134.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[25, 45, 0.5555555555555556, 0.3799361288787644, 0.7202291786230226, 0.4135697335177666, 0.7026490866732611, 0.43600244448868875, 0.6700563504951825, 0.45322713629346756, 0.6556422188566775]]} \ No newline at end of file diff --git a/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/0_eval_0_880040d892346c7fbdeb.mp4 b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/0_eval_0_880040d892346c7fbdeb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..292e92b727b3520b24cd1261f172cc754a0cbf1b --- /dev/null +++ b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/0_eval_0_880040d892346c7fbdeb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:880040d892346c7fbdeb4b37f1632611870a673ca2eaf223b34f6e2393428aa5 +size 1440847 diff --git a/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/1_eval_1_6d027f920cfefcbf4542.mp4 b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/1_eval_1_6d027f920cfefcbf4542.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..901fa41b06b1be66026883e2dfed989efd4585e5 --- /dev/null +++ b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/1_eval_1_6d027f920cfefcbf4542.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d027f920cfefcbf4542375854542541e64d2284a4aeec23e11720b75b6aca53 +size 1220755 diff --git a/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/2_eval_2_7105aabd534b6cd8cb8e.mp4 b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/2_eval_2_7105aabd534b6cd8cb8e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..33485d00d3a7790a0dddff3f6be9956674e30d95 --- /dev/null +++ b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/2_eval_2_7105aabd534b6cd8cb8e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7105aabd534b6cd8cb8e49c2530d962bf85f4c2c06674dfa2012c7e3aed5f567 +size 873820 diff --git a/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/3_eval_3_f0346db17ff10377ae08.mp4 b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/3_eval_3_f0346db17ff10377ae08.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0c0ba9783fb64bf2f35d64d41d25547b334c00df --- /dev/null +++ b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/3_eval_3_f0346db17ff10377ae08.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0346db17ff10377ae0825e93b1be0b59b49cff20bee80a503b2e42c8786c237 +size 949006 diff --git a/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/4_eval_4_77f21e81cd3b0f937629.mp4 b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/4_eval_4_77f21e81cd3b0f937629.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c68fdeeba854234f5d770688a33dd9dfb9be14de --- /dev/null +++ b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/4_eval_4_77f21e81cd3b0f937629.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77f21e81cd3b0f937629b2dc7a6931d3c6efa5c1c524b45ce54f6cdbba41c392 +size 840784 diff --git a/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a42634c877363988658b23775c1f464c7e88724c --- /dev/null +++ b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log @@ -0,0 +1,321 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735748170.408949]: resetting environment +[INFO] [1735748170.417089]: cleared current plan +[INFO] [1735748170.418247]: moving to home +[INFO] [1735748171.420377]: reached home +[INFO] [1735748171.421598]: reset action history +[INFO] [1735748174.182229]: environment reset +Starting episode 0 +[INFO] [1735748174.186502]: resetting environment +[INFO] [1735748174.191057]: cleared current plan +[INFO] [1735748174.192165]: moving to home +[INFO] [1735748175.194732]: reached home +[INFO] [1735748175.195976]: reset action history +[INFO] [1735748177.953530]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735748180.360701]: resetting environment +[INFO] [1735748180.364911]: cleared current plan +[INFO] [1735748180.366518]: moving to home +[INFO] [1735748181.369284]: reached home +[INFO] [1735748181.370542]: reset action history +[INFO] [1735748184.079120]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/episode_rosbags/episode_0_2025-01-01-11-16-24.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1791650.log +started roslaunch server http://158.130.50.37:46351/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1791930] +process[print_text-2]: started with pid [1791931] +process[rosbag_record-3]: started with pid [1791955] +started bagging! +For topic gripper_width: timestamp difference is -107529046 for nearest: 1735748192478632806 - target: 1735748192586161852 at idx 110 +For topic gripper_width: timestamp difference is 101584095 for nearest: 1735748197045325417 - target: 1735748196943741322 at idx 170 +For topic gripper_width: timestamp difference is 108604915 for nearest: 1735748200445297725 - target: 1735748200336692810 at idx 220 +[INFO] [1735748218.899410]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735748219.628775]: Stored episode 1. +[INFO] [1735748219.629043]: Saving video... +[INFO] [1735748219.822645]: Video saved! +Starting episode 1 +[INFO] [1735748219.840166]: resetting environment +[INFO] [1735748219.849851]: cleared current plan +[INFO] [1735748219.850437]: moving to home +Move to target timeout: 5.001135587692261. Trans error: 0.008838632144033909. Rot error: 1.4591196291806068 +Move to target timeout: 5.099745273590088. Trans error: 0.006131914774654496. Rot error: 0.932373019635604 +[INFO] [1735748230.953287]: reached home +[INFO] [1735748230.953716]: reset action history +[INFO] [1735748233.664262]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735748238.989854]: resetting environment +[INFO] [1735748238.996392]: cleared current plan +[INFO] [1735748238.997391]: moving to home +Move to target timeout: 5.001026153564453. Trans error: 0.005976889748126268. Rot error: 0.5748725478050736 +Move to target timeout: 5.09917688369751. Trans error: 0.006061893426204409. Rot error: 1.3879738914133803 +[INFO] [1735748250.100143]: reached home +[INFO] [1735748250.100558]: reset action history +[INFO] [1735748252.808176]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/episode_rosbags/episode_1_2025-01-01-11-17-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1791650.log +started roslaunch server http://158.130.50.37:39979/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1792169] +process[print_text-5]: started with pid [1792170] +process[rosbag_record-6]: started with pid [1792194] +started bagging! +preempting episode +[INFO] [1735748281.521514]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735748282.259402]: Stored episode 2. +[INFO] [1735748282.260287]: Saving video... +[WARN] [1735748282.732359]: Plan exhausted +[WARN] [1735748282.773275]: Plan exhausted +[WARN] [1735748282.812328]: Plan exhausted +[WARN] [1735748282.852547]: Plan exhausted +[WARN] [1735748282.892669]: Plan exhausted +[WARN] [1735748282.932795]: Plan exhausted +[WARN] [1735748282.974094]: Plan exhausted +[INFO] [1735748282.938640]: Video saved! +Starting episode 2 +[INFO] [1735748282.967205]: resetting environment +[INFO] [1735748282.986794]: cleared current plan +[INFO] [1735748282.987045]: moving to home +Move to target timeout: 5.005028247833252. Trans error: 0.008890754543244839. Rot error: 1.3459919055225984 +Move to target timeout: 5.093645095825195. Trans error: 0.00615940767165903. Rot error: 1.0177973605460577 +[INFO] [1735748294.096968]: reached home +[INFO] [1735748294.098890]: reset action history +[INFO] [1735748296.826676]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735748305.701602]: resetting environment +[INFO] [1735748305.709994]: cleared current plan +[INFO] [1735748305.710192]: moving to home +Move to target timeout: 5.000660419464111. Trans error: 0.006055253557860851. Rot error: 0.6099689776008672 +Move to target timeout: 5.099881649017334. Trans error: 0.00623075105704447. Rot error: 1.4732083685996602 +[INFO] [1735748316.812771]: reached home +[INFO] [1735748316.813999]: reset action history +[INFO] [1735748319.529323]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/episode_rosbags/episode_2_2025-01-01-11-18-39.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1791650.log +started roslaunch server http://158.130.50.37:40275/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1792334] +process[print_text-8]: started with pid [1792335] +process[rosbag_record-9]: started with pid [1792336] +started bagging! +[WARN] [1735748339.817801]: Plan exhausted +[WARN] [1735748339.857797]: Plan exhausted +[WARN] [1735748339.899829]: Plan exhausted +[WARN] [1735748339.939361]: Plan exhausted +[WARN] [1735748339.978243]: Plan exhausted +[WARN] [1735748340.017981]: Plan exhausted +[WARN] [1735748340.060715]: Plan exhausted +[WARN] [1735748340.098211]: Plan exhausted +[WARN] [1735748340.138333]: Plan exhausted +preempting episode +[INFO] [1735748338.721431]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1735748339.544769]: Stored episode 3. +[INFO] [1735748339.549518]: Saving video... +[INFO] [1735748340.120340]: Video saved! +Starting episode 3 +[INFO] [1735748340.164281]: resetting environment +[INFO] [1735748340.175352]: cleared current plan +[INFO] [1735748340.175588]: moving to home +Move to target timeout: 5.001964569091797. Trans error: 0.007323396857827902. Rot error: 0.8124749496484469 +Move to target timeout: 5.089974641799927. Trans error: 0.0062276403235625305. Rot error: 1.0201832571920026 +[INFO] [1735748351.281898]: reached home +[INFO] [1735748351.296100]: reset action history +[INFO] [1735748354.138278]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735748363.437344]: resetting environment +[INFO] [1735748363.454868]: cleared current plan +[INFO] [1735748363.456151]: moving to home +Move to target timeout: 5.002365827560425. Trans error: 0.0061242724768817425. Rot error: 0.6554004143655326 +Move to target timeout: 5.097265243530273. Trans error: 0.0063045778368498616. Rot error: 1.5040799780855816 +[INFO] [1735748374.566859]: reached home +[INFO] [1735748374.570183]: reset action history +[INFO] [1735748377.351165]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/episode_rosbags/episode_3_2025-01-01-11-19-37.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1791650.log +started roslaunch server http://158.130.50.37:41061/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1792498] +process[print_text-11]: started with pid [1792499] +process[rosbag_record-12]: started with pid [1792500] +started bagging! +For topic gripper_width: timestamp difference is 114679208 for nearest: 1735748378945298305 - target: 1735748378830619097 at idx 24 +preempting episode +[WARN] [1735748399.779158]: Plan exhausted +[WARN] [1735748399.822658]: Plan exhausted +[WARN] [1735748399.857878]: Plan exhausted +[WARN] [1735748399.898636]: Plan exhausted +[WARN] [1735748399.938457]: Plan exhausted +[WARN] [1735748399.981028]: Plan exhausted +[WARN] [1735748400.020637]: Plan exhausted +[INFO] [1735748398.643814]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit[pub_text-10] killing on exit +[INFO] [1735748399.448036]: Stored episode 4. +[INFO] [1735748399.452930]: Saving video... +[INFO] [1735748399.948687]: Video saved! +Starting episode 4 +[INFO] [1735748400.004097]: resetting environment +[INFO] [1735748400.036541]: cleared current plan +[INFO] [1735748400.037344]: moving to home +Move to target timeout: 5.000763654708862. Trans error: 0.008772829547524452. Rot error: 1.3026454323382324 +Move to target timeout: 5.088043928146362. Trans error: 0.006251573491301536. Rot error: 1.0002573084236412 +[INFO] [1735748411.151457]: reached home +[INFO] [1735748411.158752]: reset action history +[INFO] [1735748414.002253]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735748418.622725]: resetting environment +[INFO] [1735748418.635270]: cleared current plan +[INFO] [1735748418.635470]: moving to home +Move to target timeout: 5.001108169555664. Trans error: 0.006102216895669699. Rot error: 0.644562108212269 +Move to target timeout: 5.000135660171509. Trans error: 0.006289899803405254. Rot error: 1.4750892310846697 +[INFO] [1735748429.644078]: reached home +[INFO] [1735748429.650262]: reset action history +[INFO] [1735748432.441614]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/episode_rosbags/episode_4_2025-01-01-11-20-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1791650.log +started roslaunch server http://158.130.50.37:45967/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1792653] +process[print_text-14]: started with pid [1792654] +process[rosbag_record-15]: started with pid [1792655] +started bagging! +For topic gripper_width: timestamp difference is 100371648 for nearest: 1735748433645293523 - target: 1735748433544921875 at idx 21 +For topic gripper_width: timestamp difference is 103450590 for nearest: 1735748435145376259 - target: 1735748435041925669 at idx 40 +For topic gripper_width: timestamp difference is 104501261 for nearest: 1735748438045295014 - target: 1735748437940793753 at idx 79 +For topic gripper_width: timestamp difference is 103224427 for nearest: 1735748439345295817 - target: 1735748439242071390 at idx 96 +[WARN] [1735748452.146240]: Plan exhausted +[WARN] [1735748452.177793]: Plan exhausted +[WARN] [1735748452.217981]: Plan exhausted +[WARN] [1735748452.259292]: Plan exhausted +[WARN] [1735748452.297789]: Plan exhausted +[WARN] [1735748452.344552]: Plan exhausted +[WARN] [1735748452.381386]: Plan exhausted +preempting episode +[INFO] [1735748451.033251]: Storing episode... +[pub_text-13] killing on exit +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[INFO] [1735748451.796173]: Stored episode 5. +[INFO] [1735748451.799370]: Saving video... +[INFO] [1735748452.310623]: Video saved! +[INFO] [1735748452.388131]: resetting environment +[INFO] [1735748452.421582]: cleared current plan +[INFO] [1735748452.423684]: moving to home +[WARN] [1735748452.418076]: Plan exhausted +Move to target timeout: 5.005708456039429. Trans error: 0.00884992815554142. Rot error: 1.3392898839916587 +Move to target timeout: 5.089993476867676. Trans error: 0.0061534030652596414. Rot error: 1.002968256754506 +[INFO] [1735748463.533206]: reached home +[INFO] [1735748463.539384]: reset action history +[INFO] [1735748466.403764]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/requirements.txt b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-metadata.json b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..26aa0e3f655cd601f5f4f2ed4b6e1722158df50d --- /dev/null +++ b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T16:16:04.523898", + "startedAt": "2025-01-01T16:16:04.043345", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.45725, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.976, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 688.4198722839355 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..47c176ce8d8738c76fab0d1b81a51e5f18b2ff92 --- /dev/null +++ b/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "880040d892346c7fbdeb4b37f1632611870a673ca2eaf223b34f6e2393428aa5", "size": 1440847, "path": "media/videos/eval/0_eval_0_880040d892346c7fbdeb.mp4"}, "global_step": 462, "_timestamp": 1735748474.9218023, "_runtime": 310.8689033985138, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "6d027f920cfefcbf4542375854542541e64d2284a4aeec23e11720b75b6aca53", "size": 1220755, "path": "media/videos/eval/1_eval_1_6d027f920cfefcbf4542.mp4"}, "eval/num_success": 0.0, "episode": 45.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "7105aabd534b6cd8cb8e49c2530d962bf85f4c2c06674dfa2012c7e3aed5f567", "size": 873820, "path": "media/videos/eval/2_eval_2_7105aabd534b6cd8cb8e.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "f0346db17ff10377ae0825e93b1be0b59b49cff20bee80a503b2e42c8786c237", "size": 949006, "path": "media/videos/eval/3_eval_3_f0346db17ff10377ae08.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "77f21e81cd3b0f937629b2dc7a6931d3c6efa5c1c524b45ce54f6cdbba41c392", "size": 840784, "path": "media/videos/eval/4_eval_4_77f21e81cd3b0f937629.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://eyla1u1atg0gb23bsde78kfchqyoaiqczjux6uhvwpedjtht5fdngw0jtzotfhijfgkgqaoacytm3181ji2x6qczg3crkwg6r1blohftypo24irunz7o646wld9nikov/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://aoli6emoeu5iucfpoz5zl8hez41q5n4f1yh1nhb06a87r4gh3g1cihyfbq2m8thaa65ooccyf6z1y5a281g8sy1c9o0xpg7w0lwlsxiw97neni6h2cu0potbzfjla3bk:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 25, "total_num_episodes": 45, "total_success_rate": 0.5555555555555556, "total_success_rate_lb_0.01": 0.3799361288787644, "total_success_rate_ub_0.01": 0.7202291786230226, "total_success_rate_lb_0.025": 0.4135697335177666, "total_success_rate_ub_0.025": 0.7026490866732611, "total_success_rate_lb_0.05": 0.43600244448868875, "total_success_rate_ub_0.05": 0.6700563504951825, "total_success_rate_lb_0.1": 0.45322713629346756, "total_success_rate_ub_0.1": 0.6556422188566775, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "a25831f94f1c283f8134e49db0b8731362062564076bcb607caed0bee7f6db6b", "size": 522, "artifact_path": "wandb-client-artifact://myp107zqfjzh4yz0cq07y3cz3mq4wth5jbokonrnpfspyg8l275en8slud1pqq6dd2lwt5crvgqavzg3adymkvbmmca7poulodlqz9v9mv3qo46krshf0zpjavq73i7d/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://7yawh2kdl6fv7j249ijwa6iqa5q0vfl795rww51wmjbxnp9xzp5uh12c09zlz5dnnpx2go3it5jr09cgfevf5h5757a2ijm0nzypnzjjswbjfhmlitzw65v74j48uqc2:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_a25831f94f1c283f8134.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 338}, "eval/total_success_rate": 0.5555555820465088, "eval/total_success_rate_lb_0.01": 0.3799361288547516, "eval/total_success_rate_ub_0.01": 0.7202291488647461, "eval/total_success_rate_lb_0.025": 0.41356974840164185, "eval/total_success_rate_ub_0.025": 0.7026490569114685, "eval/total_success_rate_lb_0.05": 0.4360024333000183, "eval/total_success_rate_ub_0.05": 0.6700563430786133, "eval/total_success_rate_lb_0.1": 0.45322713255882263, "eval/total_success_rate_ub_0.1": 0.6556422114372253} \ No newline at end of file diff --git a/lydwde6h/wandb/run-20250101_111604-lydwde6h/logs/debug-internal.log b/lydwde6h/wandb/run-20250101_111604-lydwde6h/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..358074d761076bdee4378040f931d18d9c7baf5c --- /dev/null +++ b/lydwde6h/wandb/run-20250101_111604-lydwde6h/logs/debug-internal.log @@ -0,0 +1,883 @@ +2025-01-01 11:16:04,052 INFO StreamThr :1791826 [internal.py:wandb_internal():85] W&B internal server running at pid: 1791826, started at: 2025-01-01 11:16:04.052436 +2025-01-01 11:16:04,053 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status +2025-01-01 11:16:04,065 INFO WriterThread:1791826 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/run-lydwde6h.wandb +2025-01-01 11:16:04,065 DEBUG SenderThread:1791826 [sender.py:send():379] send: header +2025-01-01 11:16:04,072 DEBUG SenderThread:1791826 [sender.py:send():379] send: run +2025-01-01 11:16:04,268 INFO SenderThread:1791826 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files +2025-01-01 11:16:04,268 INFO SenderThread:1791826 [sender.py:_start_run_threads():1188] run started: lydwde6h with start time 1735748164.052899 +2025-01-01 11:16:04,285 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 11:16:04,286 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: check_version +2025-01-01 11:16:04,505 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 11:16:04,510 DEBUG HandlerThread:1791826 [system_info.py:__init__():26] System info init +2025-01-01 11:16:04,510 DEBUG HandlerThread:1791826 [system_info.py:__init__():41] System info init done +2025-01-01 11:16:04,510 INFO HandlerThread:1791826 [system_monitor.py:start():194] Starting system monitor +2025-01-01 11:16:04,510 INFO SystemMonitor:1791826 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 11:16:04,511 INFO HandlerThread:1791826 [system_monitor.py:probe():214] Collecting system info +2025-01-01 11:16:04,511 INFO SystemMonitor:1791826 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 11:16:04,512 INFO SystemMonitor:1791826 [interfaces.py:start():188] Started disk monitoring +2025-01-01 11:16:04,513 INFO SystemMonitor:1791826 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 11:16:04,513 INFO SystemMonitor:1791826 [interfaces.py:start():188] Started memory monitoring +2025-01-01 11:16:04,513 INFO SystemMonitor:1791826 [interfaces.py:start():188] Started network monitoring +2025-01-01 11:16:04,523 DEBUG HandlerThread:1791826 [system_info.py:probe():152] Probing system +2025-01-01 11:16:04,525 DEBUG HandlerThread:1791826 [system_info.py:_probe_git():137] Probing git +2025-01-01 11:16:04,529 DEBUG HandlerThread:1791826 [system_info.py:_probe_git():145] Probing git done +2025-01-01 11:16:04,529 DEBUG HandlerThread:1791826 [system_info.py:probe():200] Probing system done +2025-01-01 11:16:04,529 DEBUG HandlerThread:1791826 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T16:16:04.523898', 'startedAt': '2025-01-01T16:16:04.043345', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.45725, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.976, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 688.4198722839355}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 11:16:04,530 INFO HandlerThread:1791826 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 11:16:04,530 INFO HandlerThread:1791826 [system_monitor.py:probe():227] Publishing system info +2025-01-01 11:16:04,530 DEBUG HandlerThread:1791826 [system_info.py:_save_code():44] Saving code +2025-01-01 11:16:04,535 DEBUG HandlerThread:1791826 [system_info.py:_save_code():67] Saving code done +2025-01-01 11:16:04,535 DEBUG HandlerThread:1791826 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 11:16:04,599 DEBUG HandlerThread:1791826 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 11:16:04,599 INFO HandlerThread:1791826 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 11:16:04,602 DEBUG SenderThread:1791826 [sender.py:send():379] send: files +2025-01-01 11:16:04,602 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 11:16:04,603 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 11:16:04,603 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 11:16:04,844 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/wasgjph7-wandb-metadata.json +2025-01-01 11:16:04,849 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 11:16:04,849 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: python_packages +2025-01-01 11:16:04,850 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:16:04,850 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:16:04,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:04,852 INFO HandlerThread:1791826 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/tb" + save: true +} +control { + relay_id: "22470aca6a1243059732c3d7d33e83c8" +} +_info { + stream_id: "lydwde6h" +} + +2025-01-01 11:16:04,853 INFO wandb-upload_1:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/40lgz95o-code/FISH/eval_robot.py +2025-01-01 11:16:04,853 DEBUG HandlerThread:1791826 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 11:16:04,864 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/1n6sm2cn-diff.patch +2025-01-01 11:16:04,909 DEBUG SenderThread:1791826 [sender.py:send():379] send: telemetry +2025-01-01 11:16:04,909 DEBUG SenderThread:1791826 [sender.py:send():379] send: tbrecord +2025-01-01 11:16:04,910 DEBUG SenderThread:1791826 [sender.py:send():379] send: files +2025-01-01 11:16:04,910 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 with policy live +2025-01-01 11:16:05,095 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/nylvs93l-events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:16:05,269 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/code/FISH/eval_robot.py +2025-01-01 11:16:05,269 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:16:05,269 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/requirements.txt +2025-01-01 11:16:05,270 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/diff.patch +2025-01-01 11:16:05,270 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-metadata.json +2025-01-01 11:16:05,270 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:05,270 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/code/FISH +2025-01-01 11:16:05,270 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/code +2025-01-01 11:16:05,850 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:06,850 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:07,268 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:07,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:08,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:09,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:09,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:10,408 DEBUG SenderThread:1791826 [sender.py:send():379] send: config +2025-01-01 11:16:10,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:11,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:12,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:13,270 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:13,852 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:14,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:15,195 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:15,271 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:15,852 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:16,852 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:17,271 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:17,852 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:18,852 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:19,272 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:19,850 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:16:19,850 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:16:19,893 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:20,360 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:20,853 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:21,273 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:21,853 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:22,853 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:23,273 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:23,853 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:24,853 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:25,274 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:25,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:25,855 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:26,855 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:27,855 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:28,855 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:29,856 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:30,855 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:31,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:31,856 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:32,856 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:33,857 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:34,850 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:16:34,850 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:16:34,893 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:35,276 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:35,857 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:36,857 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:36,907 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:37,277 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/config.yaml +2025-01-01 11:16:37,857 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:38,857 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:39,278 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:39,858 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:40,857 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:41,278 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:16:41,858 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:42,489 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:42,858 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:43,858 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:44,860 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:45,860 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:46,860 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:47,854 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:47,860 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:48,860 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:49,850 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:16:49,850 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:16:49,897 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:50,860 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:51,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:52,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:52,968 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:53,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:54,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:55,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:56,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:57,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:58,854 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:16:58,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:16:59,837 DEBUG SenderThread:1791826 [sender.py:send():379] send: files +2025-01-01 11:16:59,838 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_880040d892346c7fbdeb.mp4 with policy now +2025-01-01 11:16:59,838 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:16:59,839 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:16:59,839 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:16:59,840 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:16:59,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:00,099 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/9gy5yukg-media/videos/eval/0_eval_0_880040d892346c7fbdeb.mp4 +2025-01-01 11:17:00,283 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/0_eval_0_880040d892346c7fbdeb.mp4 +2025-01-01 11:17:00,283 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:17:00,284 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media +2025-01-01 11:17:00,284 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval +2025-01-01 11:17:00,284 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos +2025-01-01 11:17:00,576 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/16cnv1b2-media/videos/eval/0_eval_0_880040d892346c7fbdeb.mp4 +2025-01-01 11:17:00,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:01,284 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:17:01,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:02,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:03,854 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:03,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:04,514 DEBUG SystemMonitor:1791826 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 11:17:04,514 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:17:04,850 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:17:04,851 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:17:04,897 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:05,864 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:06,864 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:07,286 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:17:07,863 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:08,863 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:08,950 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:09,864 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:10,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:11,287 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:17:11,864 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:12,864 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:13,287 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:17:13,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:14,709 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:14,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:15,288 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:17:15,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:16,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:17,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:18,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:19,290 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:17:19,556 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/3pu9iyuc-events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:17:19,850 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:17:19,851 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:17:19,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:19,995 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:20,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:21,290 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:17:21,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:22,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:23,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:24,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:25,292 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:17:25,855 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:25,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:26,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:27,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:28,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:29,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:30,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:31,141 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:31,294 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:17:31,867 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:32,867 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:33,867 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:34,515 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:17:34,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:17:34,851 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:17:34,897 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:35,295 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:17:35,867 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:36,867 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:36,989 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:37,867 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:38,867 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:39,867 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:40,868 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:41,868 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:42,855 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:42,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:43,869 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:44,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:45,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:46,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:47,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:48,855 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:48,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:49,851 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:17:49,851 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:17:49,893 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:50,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:51,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:52,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:53,871 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:53,912 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:54,871 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:55,871 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:56,871 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:57,871 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:58,871 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:17:59,855 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:17:59,872 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:00,871 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:01,871 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:02,876 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:02,960 DEBUG SenderThread:1791826 [sender.py:send():379] send: files +2025-01-01 11:18:02,960 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6d027f920cfefcbf4542.mp4 with policy now +2025-01-01 11:18:02,963 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:18:02,963 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:18:02,964 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:18:02,964 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:18:03,210 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/o4jnrdip-media/videos/eval/1_eval_1_6d027f920cfefcbf4542.mp4 +2025-01-01 11:18:03,304 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:18:03,304 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:03,304 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/1_eval_1_6d027f920cfefcbf4542.mp4 +2025-01-01 11:18:03,304 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval +2025-01-01 11:18:03,556 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/ao2hm548-media/videos/eval/1_eval_1_6d027f920cfefcbf4542.mp4 +2025-01-01 11:18:03,879 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:04,515 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:18:04,859 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:18:04,859 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:18:04,901 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:04,927 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:05,304 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:05,878 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:06,881 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:07,878 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:08,879 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:09,306 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:09,880 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:10,865 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:10,882 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:11,879 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:12,887 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:13,884 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:14,891 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:15,308 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:15,893 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:16,827 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:16,887 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:17,887 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:18,887 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:19,309 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:19,856 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:18:19,856 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:18:19,901 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:20,887 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:21,888 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:22,003 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:22,888 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:23,889 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:24,888 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:25,888 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:26,296 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:18:26,296 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:18:26,298 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:18:26,311 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:18:26,311 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:18:26,560 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/16zi9lt4-events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:18:26,888 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:27,298 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:27,311 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:27,889 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:28,888 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:29,888 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:30,890 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:31,889 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:32,753 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:32,889 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:33,314 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:33,889 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:34,516 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:18:34,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:18:34,863 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:18:34,909 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:35,890 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:36,890 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:37,315 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:37,853 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:37,890 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:38,891 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:39,316 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:39,891 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:40,891 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:41,316 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:18:41,891 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:42,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:42,891 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:43,891 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:44,892 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:45,892 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:46,892 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:47,892 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:48,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:48,893 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:49,857 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:18:49,857 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:18:49,901 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:50,893 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:51,893 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:52,894 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:53,894 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:53,943 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:54,894 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:55,894 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:56,895 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:57,894 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:58,894 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:18:59,546 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:18:59,899 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:00,159 DEBUG SenderThread:1791826 [sender.py:send():379] send: files +2025-01-01 11:19:00,159 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_7105aabd534b6cd8cb8e.mp4 with policy now +2025-01-01 11:19:00,160 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:19:00,161 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:19:00,161 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:19:00,163 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:19:00,323 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:19:00,323 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/2_eval_2_7105aabd534b6cd8cb8e.mp4 +2025-01-01 11:19:00,323 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval +2025-01-01 11:19:00,396 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/14weo606-media/videos/eval/2_eval_2_7105aabd534b6cd8cb8e.mp4 +2025-01-01 11:19:00,638 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/ixt8ygs3-media/videos/eval/2_eval_2_7105aabd534b6cd8cb8e.mp4 +2025-01-01 11:19:00,898 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:01,323 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:01,898 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:02,907 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:03,903 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:04,516 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:19:04,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:19:04,866 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:19:04,913 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:04,990 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:05,912 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:06,912 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:07,325 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:07,913 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:08,915 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:09,917 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:10,277 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:10,922 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:11,326 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:11,930 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:12,932 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:13,327 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:13,930 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:14,931 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:15,328 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:15,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:15,933 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:16,935 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:17,938 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:18,941 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:19,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:19:19,861 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:19:19,941 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:20,940 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:20,970 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:21,936 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:22,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:23,940 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:24,144 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:19:24,145 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:19:24,146 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:19:24,331 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:19:24,331 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:19:24,948 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:25,331 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:25,945 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:26,878 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:26,945 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:27,949 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:28,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:29,332 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:29,943 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:30,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:31,883 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:31,943 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:32,945 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:33,950 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:34,517 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:19:34,866 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:19:34,867 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:19:34,945 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:35,334 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:35,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:36,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:36,965 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:37,945 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:38,945 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:39,336 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:39,945 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:40,945 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:41,336 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:41,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:42,873 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:42,945 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:43,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:44,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:45,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:46,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:47,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:48,873 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:48,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:49,861 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:19:49,861 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:19:49,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:50,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:51,946 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:52,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:53,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:53,973 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:54,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:55,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:56,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:57,947 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:58,949 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:19:59,342 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:19:59,449 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:19:59,959 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:00,025 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:20:00,025 DEBUG SenderThread:1791826 [sender.py:send():379] send: files +2025-01-01 11:20:00,026 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f0346db17ff10377ae08.mp4 with policy now +2025-01-01 11:20:00,026 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:20:00,026 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:20:00,026 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:20:00,269 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/2yzey7os-media/videos/eval/3_eval_3_f0346db17ff10377ae08.mp4 +2025-01-01 11:20:00,343 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:20:00,343 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/3_eval_3_f0346db17ff10377ae08.mp4 +2025-01-01 11:20:00,343 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval +2025-01-01 11:20:00,599 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/shkrhvhq-media/videos/eval/3_eval_3_f0346db17ff10377ae08.mp4 +2025-01-01 11:20:00,961 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:01,343 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:01,958 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:02,965 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:03,963 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:04,517 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:20:04,518 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:04,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:20:04,862 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:20:04,962 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:05,976 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:06,967 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:07,345 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:07,974 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:08,980 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:09,880 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:09,967 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:10,985 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:11,346 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:11,982 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:12,987 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:13,347 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:13,988 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:14,889 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:14,991 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:15,348 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:15,989 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:16,995 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:17,991 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:18,990 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:19,211 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:20:19,211 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:20:19,211 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:20:19,349 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:20:19,349 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:20:19,349 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:19,594 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/bg9vir2h-events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:20:19,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:20:19,862 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:20:19,953 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:19,999 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:20,993 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:21,996 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:22,994 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:24,001 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:25,002 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:25,351 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:25,887 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:26,006 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:27,007 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:28,005 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:29,007 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:29,352 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:30,008 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:31,012 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:31,353 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:31,882 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:32,019 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:33,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:33,354 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:34,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:34,518 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:20:34,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:20:34,862 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:20:35,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:35,355 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:36,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:37,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:37,201 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:37,355 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:38,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:39,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:39,356 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:40,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:41,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:41,357 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:42,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:42,881 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:43,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:44,013 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:45,014 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:46,014 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:47,014 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:47,881 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:48,014 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:49,014 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:49,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:20:49,862 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:20:50,014 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:51,017 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:52,022 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:52,361 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/4_eval_4_77f21e81cd3b0f937629.mp4 +2025-01-01 11:20:52,361 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval +2025-01-01 11:20:52,382 DEBUG SenderThread:1791826 [sender.py:send():379] send: files +2025-01-01 11:20:52,382 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_77f21e81cd3b0f937629.mp4 with policy now +2025-01-01 11:20:52,387 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:20:52,388 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:20:52,388 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:20:52,388 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:20:52,618 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/gkncnkvn-media/videos/eval/4_eval_4_77f21e81cd3b0f937629.mp4 +2025-01-01 11:20:53,019 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:53,361 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:53,361 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/4_eval_4_77f21e81cd3b0f937629.mp4 +2025-01-01 11:20:53,361 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:20:53,424 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:53,637 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/zl7rilq9-media/videos/eval/4_eval_4_77f21e81cd3b0f937629.mp4 +2025-01-01 11:20:54,028 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:55,023 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:55,362 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:20:56,040 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:57,024 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:58,031 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:58,473 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:20:59,036 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:20:59,363 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:21:00,045 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:01,051 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:02,050 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:03,053 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:03,364 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:21:03,534 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:04,049 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:04,518 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:21:04,877 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:21:04,877 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:21:05,056 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:05,365 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:21:06,051 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:07,052 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:07,366 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:21:08,054 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:09,057 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:09,449 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:10,075 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:11,061 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:12,068 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:13,070 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:14,064 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:14,277 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:21:14,278 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:14,278 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:14,368 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:21:14,368 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:21:14,425 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:21:14,426 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:21:14,623 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:21:14,847 INFO wandb-upload_0:1791826 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpd19lb3vg +2025-01-01 11:21:15,019 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:21:15,028 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:21:15,056 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:15,256 INFO SenderThread:1791826 [sender.py:send_request_log_artifact():1525] logged artifact run-lydwde6h-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY4MzQxNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MTg2Ng==', 'latestArtifact': None}} +2025-01-01 11:21:15,256 DEBUG SenderThread:1791826 [sender.py:send():379] send: files +2025-01-01 11:21:15,256 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-01 11:21:15,256 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:15,256 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:21:15,256 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,257 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,257 DEBUG SenderThread:1791826 [sender.py:send():379] send: telemetry +2025-01-01 11:21:15,257 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,258 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,258 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,260 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,260 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,261 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,261 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,261 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,261 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,262 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,262 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,262 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,262 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,263 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,263 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,263 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,263 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,263 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,264 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,264 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,264 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:15,265 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:15,265 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:21:15,369 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:21:15,369 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:21:15,369 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:21:15,369 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/table/eval/total_success_rate_ci_6_a25831f94f1c283f8134.table.json +2025-01-01 11:21:15,369 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:21:15,369 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media +2025-01-01 11:21:15,370 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/table/eval +2025-01-01 11:21:15,370 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/table +2025-01-01 11:21:15,474 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/m2mqyv95-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:21:15,582 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/nducn3wp-media/table/eval/total_success_rate_ci_6_a25831f94f1c283f8134.table.json +2025-01-01 11:21:15,711 INFO wandb-upload_1:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/cf06v01f-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:21:15,870 INFO wandb-upload_2:1791826 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmexu6gcz +2025-01-01 11:21:16,056 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:16,267 INFO SenderThread:1791826 [sender.py:send_request_log_artifact():1525] logged artifact run-lydwde6h-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY4MzQyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MTg2Nw==', 'latestArtifact': None}} +2025-01-01 11:21:16,267 DEBUG SenderThread:1791826 [sender.py:send():379] send: files +2025-01-01 11:21:16,267 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_a25831f94f1c283f8134.table.json with policy now +2025-01-01 11:21:16,267 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:21:16,267 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:16,268 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:16,369 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:21:16,504 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/lquv2xxs-media/table/eval/total_success_rate_ci_6_a25831f94f1c283f8134.table.json +2025-01-01 11:21:17,056 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:17,370 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:21:18,056 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:19,057 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:19,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:21:19,862 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:21:20,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:21,004 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:21,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:22,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:23,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:24,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:25,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:26,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:26,891 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:27,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:28,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:29,058 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:30,059 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:31,059 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:31,894 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:32,059 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:32,375 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/config.yaml +2025-01-01 11:21:33,059 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:34,059 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:34,519 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:21:34,862 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:21:34,862 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:21:35,059 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:36,059 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:37,060 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:37,933 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:38,060 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:39,060 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:40,060 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:41,060 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:42,060 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:42,941 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:42,942 DEBUG SenderThread:1791826 [sender.py:send():379] send: exit +2025-01-01 11:21:42,942 INFO SenderThread:1791826 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 11:21:42,942 INFO SenderThread:1791826 [sender.py:send_exit():588] handling runtime: 338 +2025-01-01 11:21:42,942 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:42,942 INFO SenderThread:1791826 [sender.py:send_exit():594] send defer +2025-01-01 11:21:42,942 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:42,942 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 11:21:42,943 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:42,943 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 11:21:42,943 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 1 +2025-01-01 11:21:42,943 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:42,943 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 11:21:42,943 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:42,943 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 11:21:42,943 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 2 +2025-01-01 11:21:42,943 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:42,943 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 11:21:42,943 INFO HandlerThread:1791826 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 11:21:42,944 DEBUG SystemMonitor:1791826 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 11:21:42,944 DEBUG SystemMonitor:1791826 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 11:21:42,944 INFO HandlerThread:1791826 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 11:21:42,944 INFO HandlerThread:1791826 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 11:21:42,994 INFO HandlerThread:1791826 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 11:21:42,994 INFO HandlerThread:1791826 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 11:21:42,994 INFO HandlerThread:1791826 [interfaces.py:finish():200] Joined network monitor +2025-01-01 11:21:42,995 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:42,995 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 11:21:42,995 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 3 +2025-01-01 11:21:42,995 DEBUG SenderThread:1791826 [sender.py:send():379] send: stats +2025-01-01 11:21:42,995 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:42,995 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 11:21:42,996 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:42,996 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 11:21:42,996 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 4 +2025-01-01 11:21:42,996 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:42,996 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 11:21:43,380 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:21:45,381 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:21:49,286 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:21:49,286 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:49,286 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 11:21:49,287 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:49,287 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 11:21:49,287 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 5 +2025-01-01 11:21:49,287 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:21:49,287 DEBUG SenderThread:1791826 [sender.py:send():379] send: history +2025-01-01 11:21:49,288 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:21:49,288 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:49,288 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 11:21:49,288 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:49,289 DEBUG SenderThread:1791826 [sender.py:send():379] send: summary +2025-01-01 11:21:49,292 INFO SenderThread:1791826 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:21:49,293 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:49,293 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 11:21:49,293 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 6 +2025-01-01 11:21:49,293 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:49,293 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 11:21:49,293 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:49,293 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 11:21:49,293 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 7 +2025-01-01 11:21:49,293 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:21:49,293 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:49,293 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 11:21:49,294 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:49,294 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 11:21:49,382 INFO Thread-12 :1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:21:49,943 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:21:52,947 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 8 +2025-01-01 11:21:52,947 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:21:52,948 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:52,948 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 11:21:52,948 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:52,948 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 11:21:52,948 INFO SenderThread:1791826 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 11:21:52,950 INFO SenderThread:1791826 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 11:21:52,952 INFO SenderThread:1791826 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 11:21:52,967 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 9 +2025-01-01 11:21:52,967 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:52,967 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 11:21:52,967 DEBUG SenderThread:1791826 [sender.py:send():379] send: artifact +2025-01-01 11:21:53,150 INFO SenderThread:1791826 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjk2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 11:21:53,150 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:53,150 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 11:21:53,150 INFO SenderThread:1791826 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 11:21:53,383 INFO SenderThread:1791826 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:21:53,383 INFO SenderThread:1791826 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files +2025-01-01 11:21:53,383 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/config.yaml config.yaml +2025-01-01 11:21:53,384 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json wandb-summary.json +2025-01-01 11:21:53,384 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/requirements.txt requirements.txt +2025-01-01 11:21:53,384 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-metadata.json wandb-metadata.json +2025-01-01 11:21:53,384 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:21:53,384 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log output.log +2025-01-01 11:21:53,384 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/diff.patch diff.patch +2025-01-01 11:21:53,387 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 11:21:53,389 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/1_eval_1_6d027f920cfefcbf4542.mp4 media/videos/eval/1_eval_1_6d027f920cfefcbf4542.mp4 +2025-01-01 11:21:53,389 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/4_eval_4_77f21e81cd3b0f937629.mp4 media/videos/eval/4_eval_4_77f21e81cd3b0f937629.mp4 +2025-01-01 11:21:53,389 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/0_eval_0_880040d892346c7fbdeb.mp4 media/videos/eval/0_eval_0_880040d892346c7fbdeb.mp4 +2025-01-01 11:21:53,390 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/2_eval_2_7105aabd534b6cd8cb8e.mp4 media/videos/eval/2_eval_2_7105aabd534b6cd8cb8e.mp4 +2025-01-01 11:21:53,390 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/videos/eval/3_eval_3_f0346db17ff10377ae08.mp4 media/videos/eval/3_eval_3_f0346db17ff10377ae08.mp4 +2025-01-01 11:21:53,390 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/table/eval/total_success_rate_ci_6_a25831f94f1c283f8134.table.json media/table/eval/total_success_rate_ci_6_a25831f94f1c283f8134.table.json +2025-01-01 11:21:53,390 INFO SenderThread:1791826 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:21:53,390 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 10 +2025-01-01 11:21:53,390 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:53,390 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 11:21:53,393 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:53,393 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 11:21:53,393 INFO SenderThread:1791826 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:21:53,569 INFO wandb-upload_1:1791826 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/wandb-summary.json +2025-01-01 11:21:53,571 INFO wandb-upload_2:1791826 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/config.yaml +2025-01-01 11:21:53,595 INFO wandb-upload_0:1791826 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/requirements.txt +2025-01-01 11:21:53,607 INFO wandb-upload_4:1791826 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/files/output.log +2025-01-01 11:21:53,668 INFO wandb-upload_3:1791826 [upload_job.py:push():130] Uploaded file /tmp/tmpg1yboc1jwandb/7sn15jvp-events.out.tfevents.1735748164.leonmkim-ROG-Strix-G15CS-G15CS.1791650.0 +2025-01-01 11:21:53,868 INFO Thread-11 (_thread_body):1791826 [sender.py:transition_state():614] send defer: 11 +2025-01-01 11:21:53,869 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:53,869 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 11:21:53,869 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:53,869 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 11:21:53,869 INFO SenderThread:1791826 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 11:21:53,870 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 12 +2025-01-01 11:21:53,870 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:53,870 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 11:21:53,871 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:53,871 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 11:21:53,871 INFO SenderThread:1791826 [file_stream.py:finish():601] file stream finish called +2025-01-01 11:21:53,944 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:21:54,071 INFO SenderThread:1791826 [file_stream.py:finish():605] file stream finish is done +2025-01-01 11:21:54,071 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 13 +2025-01-01 11:21:54,071 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:21:54,071 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:54,071 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 11:21:54,071 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:54,071 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 11:21:54,072 INFO SenderThread:1791826 [sender.py:transition_state():614] send defer: 14 +2025-01-01 11:21:54,072 DEBUG SenderThread:1791826 [sender.py:send():379] send: final +2025-01-01 11:21:54,072 DEBUG SenderThread:1791826 [sender.py:send():379] send: footer +2025-01-01 11:21:54,072 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:21:54,072 INFO HandlerThread:1791826 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 11:21:54,072 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: defer +2025-01-01 11:21:54,072 INFO SenderThread:1791826 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 11:21:54,073 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:21:54,073 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:21:54,073 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 11:21:54,073 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:21:54,073 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 11:21:54,074 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: server_info +2025-01-01 11:21:54,075 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 11:21:54,076 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:21:54,126 DEBUG SenderThread:1791826 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:21:54,127 INFO MainThread:1791826 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 11:21:54,127 INFO MainThread:1791826 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 11:21:54,127 INFO MainThread:1791826 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 11:21:54,128 DEBUG HandlerThread:1791826 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 11:21:54,128 INFO HandlerThread:1791826 [handler.py:finish():882] shutting down handler +2025-01-01 11:21:55,074 INFO WriterThread:1791826 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/run-lydwde6h.wandb +2025-01-01 11:21:55,127 INFO SenderThread:1791826 [sender.py:finish():1615] shutting down sender +2025-01-01 11:21:55,127 INFO SenderThread:1791826 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:21:55,127 INFO SenderThread:1791826 [file_pusher.py:join():175] waiting for file pusher diff --git a/lydwde6h/wandb/run-20250101_111604-lydwde6h/logs/debug.log b/lydwde6h/wandb/run-20250101_111604-lydwde6h/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8509aaf710a322ab8494879adc1d2b7e39992e45 --- /dev/null +++ b/lydwde6h/wandb/run-20250101_111604-lydwde6h/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-01 11:16:04,046 INFO MainThread:1791650 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 11:16:04,046 INFO MainThread:1791650 [wandb_setup.py:_flush():76] Configure stats pid to 1791650 +2025-01-01 11:16:04,046 INFO MainThread:1791650 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/settings +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/logs/debug.log +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/wandb/run-20250101_111604-lydwde6h/logs/debug-internal.log +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_init.py:init():569] calling init triggers +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h'} +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_init.py:init():619] starting backend +2025-01-01 11:16:04,047 INFO MainThread:1791650 [wandb_init.py:init():623] setting up manager +2025-01-01 11:16:04,051 INFO MainThread:1791650 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 11:16:04,052 INFO MainThread:1791650 [wandb_init.py:init():631] backend started and connected +2025-01-01 11:16:04,064 INFO MainThread:1791650 [wandb_init.py:init():720] updated telemetry +2025-01-01 11:16:04,070 INFO MainThread:1791650 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 11:16:04,285 INFO MainThread:1791650 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 11:16:04,500 INFO MainThread:1791650 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 11:16:04,500 INFO MainThread:1791650 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 11:16:04,850 INFO MainThread:1791650 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 11:16:04,850 INFO MainThread:1791650 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 11:16:04,850 INFO MainThread:1791650 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 11:16:04,850 INFO MainThread:1791650 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 11:16:04,852 INFO MainThread:1791650 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 11:16:04,852 INFO MainThread:1791650 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/lydwde6h/tb, True +2025-01-01 11:16:10,408 INFO MainThread:1791650 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-01 11:21:55,128 WARNING MsgRouterThr:1791650 [router.py:message_loop():77] message_loop has been closed diff --git a/lydwde6h/wandb/run-20250101_111604-lydwde6h/run-lydwde6h.wandb b/lydwde6h/wandb/run-20250101_111604-lydwde6h/run-lydwde6h.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ce113acb305e8c67bce2ab0468275edc582be485 Binary files /dev/null and b/lydwde6h/wandb/run-20250101_111604-lydwde6h/run-lydwde6h.wandb differ diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..61bb4bd15d092a8c7eaf261f2c43cc09d7b7390e --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:faf426f2d867a815ba337985cc656e593e710c2a666774a25c6d61620aa7c62c +size 910445958 diff --git a/snapshot_10999.pt b/snapshot_10999.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a3a05c9dd902d772b425e5e02aa532c7206a451 --- /dev/null +++ b/snapshot_10999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11b784a00fc152ebbde0131d306efd4771fc5d9d81f4ba405cf899266ad82cb1 +size 910445958 diff --git a/snapshot_11150.pt b/snapshot_11150.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a3a05c9dd902d772b425e5e02aa532c7206a451 --- /dev/null +++ b/snapshot_11150.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11b784a00fc152ebbde0131d306efd4771fc5d9d81f4ba405cf899266ad82cb1 +size 910445958 diff --git a/snapshot_11499.pt b/snapshot_11499.pt new file mode 100644 index 0000000000000000000000000000000000000000..1712f1479c5c9c296d9092449e44df2fe3e0a803 --- /dev/null +++ b/snapshot_11499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2910f7212f1effe54b67ef536e2b516268e17caa381912d7e07eb73710d5d70f +size 910445958 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..538d2bc25a2433fadd95f71203ddaaa850e96fd4 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c6b51d9fc01b2b4a9f36c343b046768805d092a2968d3a6083947df57aa4ca6 +size 910445958 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..9dc291b62f53ac5c8178597468619109c2242fcd --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48f0181a30a74431749d95dd6ce39103bc8b6e89bd0b88fb0d45e5e87487ef75 +size 910445958 diff --git a/snapshot_1500.pt b/snapshot_1500.pt new file mode 100644 index 0000000000000000000000000000000000000000..22fc7768e05058f8a587571e31260ec632228e54 --- /dev/null +++ b/snapshot_1500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cd83cea04f352658e111d088c160d545ea62c784537a244362b7338ed033a63 +size 910445448 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ff097b460f5f5bce1d3bac12b600d569f90ef26 --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1627f784070b6e0fef858379cd794b0cbd9fc8bcc6e7f468de476da8304d6ed0 +size 910445958 diff --git a/snapshot_3000.pt b/snapshot_3000.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5fad260484d92044f545609642c6d064339c2f9 --- /dev/null +++ b/snapshot_3000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b661a02c0acadc805769e60732ae42e12af181e885e34d4d719755e0ceea276 +size 910445958 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..f87c9b5f562e70991836aadf38eca79403097698 --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f69c9d755e2a8507e1cd61c2650ede17242ff0870591f9c6ce40b7646e9fbfcb +size 910445958 diff --git a/snapshot_4834.pt b/snapshot_4834.pt new file mode 100644 index 0000000000000000000000000000000000000000..f87c9b5f562e70991836aadf38eca79403097698 --- /dev/null +++ b/snapshot_4834.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f69c9d755e2a8507e1cd61c2650ede17242ff0870591f9c6ce40b7646e9fbfcb +size 910445958 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..2a4baf131f5fc42eb7f29a958d058cd1d8dc61b2 --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ba3057acbf263eabde6bfa101ea5c8ecdc830c96e4fce4a1773899ae88dc4a8 +size 910445958 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..76f778ff5c6e5a096bbbb973905ae5e0f644d9e4 --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccee1056eac2ffe36b534e8cb48529d58f292bc9ec09c736f5c53432c3c9224e +size 910445958 diff --git a/snapshot_8499.pt b/snapshot_8499.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6b914a01236ba0feff06553dcf847fcd5104c0c --- /dev/null +++ b/snapshot_8499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:816588b5f3ad23b80ceaf6557e2a48ea984c1ed2fb23755296de91311a502498 +size 910445958 diff --git a/snapshot_8749.pt b/snapshot_8749.pt new file mode 100644 index 0000000000000000000000000000000000000000..d493544d7b895efb3e677a841a92153b0f1ee910 --- /dev/null +++ b/snapshot_8749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28f9de47aa40058d19188de49198c796b1ae173f8cf3ea3dc1ddd9f1d034f5ec +size 910445958 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..94ff49fac9e1a5abb0fa124a893bacaa7137bda2 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:664eacb005b6fd593053c3f7120b5be399270f14185423b3051f186407450808 +size 910445958 diff --git a/snapshot_9999.pt b/snapshot_9999.pt new file mode 100644 index 0000000000000000000000000000000000000000..737f1c606ea2aa6fc48eb1784a344b49652e3b90 --- /dev/null +++ b/snapshot_9999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63cc3eacc3b6963c8c88559736fd8ecea075f0e1f333865967266368ca51019d +size 910445958 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ff097b460f5f5bce1d3bac12b600d569f90ef26 --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1627f784070b6e0fef858379cd794b0cbd9fc8bcc6e7f468de476da8304d6ed0 +size 910445958 diff --git a/z0y7vp39/.hydra/config.yaml b/z0y7vp39/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05485f98b3c08bece875a460fe25a0cce7504f90 --- /dev/null +++ b/z0y7vp39/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1006_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/z0y7vp39/.hydra/hydra.yaml b/z0y7vp39/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52851a007a969b611afbfd1170b32e7e79ea0a38 --- /dev/null +++ b/z0y7vp39/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/z0y7vp39/episode_rosbags/aligned_depth_to_color_K.npy b/z0y7vp39/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/z0y7vp39/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/z0y7vp39/episode_rosbags/cam_tf_world.npy b/z0y7vp39/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/z0y7vp39/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/z0y7vp39/episode_rosbags/color_K.npy b/z0y7vp39/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/z0y7vp39/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/z0y7vp39/episode_rosbags/depth_K.npy b/z0y7vp39/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/z0y7vp39/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/z0y7vp39/eval_robot.log b/z0y7vp39/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..0e69734f3250f0901ad59cd20a4970285b4c0118 --- /dev/null +++ b/z0y7vp39/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-31 21:27:25,653][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-31 21:27:25,656][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/z0y7vp39/eval_video/0_eval.mp4 b/z0y7vp39/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3ac1f78da1799e0324a44a95a1f046f40978735c --- /dev/null +++ b/z0y7vp39/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73394dae2cb5c0e3e76ddd01183d5fb1de9dd95392128b29d1c428faa8a3c842 +size 706666 diff --git a/z0y7vp39/eval_video/1_eval.mp4 b/z0y7vp39/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2bb2219d7538095fdfc950d07325cca3dbe2161c --- /dev/null +++ b/z0y7vp39/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb820ba01fc256486a6ce64bb8e09317279dd63b1bb04cdd34a8f99ac85e8747 +size 621660 diff --git a/z0y7vp39/eval_video/2_eval.mp4 b/z0y7vp39/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..125c3ea8f907fb60bc6b352247b3a9d23947e9c4 --- /dev/null +++ b/z0y7vp39/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8637db17a31d8f9e9a53089018168b3d8b4ac0608ff98f962c66d4250bf80a3f +size 689867 diff --git a/z0y7vp39/eval_video/3_eval.mp4 b/z0y7vp39/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c5f3f33e763488c891a5d8d3f36a663bfbbaf4e5 --- /dev/null +++ b/z0y7vp39/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f1bea0839448d87829bcde0ef5cbba8ece5cbb3d99968a605f93e85d36e44ce +size 1566098 diff --git a/z0y7vp39/eval_video/4_eval.mp4 b/z0y7vp39/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6fb95bc1f9658ed466df014cc7f06502d24c4e5f --- /dev/null +++ b/z0y7vp39/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b071c579be3fd81d6bdd5c557c3c3e485a6e0d9f3e8e02f49c17a533ec3a014 +size 1534428 diff --git a/z0y7vp39/tb/events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 b/z0y7vp39/tb/events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 new file mode 100644 index 0000000000000000000000000000000000000000..49313fb089398fb05c925f23a11dfcf63f81a7c9 --- /dev/null +++ b/z0y7vp39/tb/events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf62463d06f93cd0c2010f75c221cb48602a018e2f2ca05ea2e72b51a38367cf +size 6499 diff --git a/z0y7vp39/wandb/debug-internal.log b/z0y7vp39/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..df078523addccacbe9cc5f318604cc24abdc2631 --- /dev/null +++ b/z0y7vp39/wandb/debug-internal.log @@ -0,0 +1,1034 @@ +2024-12-31 21:27:31,572 INFO StreamThr :1746604 [internal.py:wandb_internal():85] W&B internal server running at pid: 1746604, started at: 2024-12-31 21:27:31.571700 +2024-12-31 21:27:31,573 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status +2024-12-31 21:27:31,584 INFO WriterThread:1746604 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/run-z0y7vp39.wandb +2024-12-31 21:27:31,584 DEBUG SenderThread:1746604 [sender.py:send():379] send: header +2024-12-31 21:27:31,592 DEBUG SenderThread:1746604 [sender.py:send():379] send: run +2024-12-31 21:27:31,853 INFO SenderThread:1746604 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files +2024-12-31 21:27:31,853 INFO SenderThread:1746604 [sender.py:_start_run_threads():1188] run started: z0y7vp39 with start time 1735698451.572232 +2024-12-31 21:27:31,870 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 21:27:31,870 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: check_version +2024-12-31 21:27:31,953 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 21:27:31,958 DEBUG HandlerThread:1746604 [system_info.py:__init__():26] System info init +2024-12-31 21:27:31,958 DEBUG HandlerThread:1746604 [system_info.py:__init__():41] System info init done +2024-12-31 21:27:31,958 INFO HandlerThread:1746604 [system_monitor.py:start():194] Starting system monitor +2024-12-31 21:27:31,958 INFO SystemMonitor:1746604 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 21:27:31,958 INFO HandlerThread:1746604 [system_monitor.py:probe():214] Collecting system info +2024-12-31 21:27:31,959 INFO SystemMonitor:1746604 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 21:27:31,959 INFO SystemMonitor:1746604 [interfaces.py:start():188] Started disk monitoring +2024-12-31 21:27:31,960 INFO SystemMonitor:1746604 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 21:27:31,960 INFO SystemMonitor:1746604 [interfaces.py:start():188] Started memory monitoring +2024-12-31 21:27:31,961 INFO SystemMonitor:1746604 [interfaces.py:start():188] Started network monitoring +2024-12-31 21:27:31,971 DEBUG HandlerThread:1746604 [system_info.py:probe():152] Probing system +2024-12-31 21:27:31,973 DEBUG HandlerThread:1746604 [system_info.py:_probe_git():137] Probing git +2024-12-31 21:27:31,977 DEBUG HandlerThread:1746604 [system_info.py:_probe_git():145] Probing git done +2024-12-31 21:27:31,977 DEBUG HandlerThread:1746604 [system_info.py:probe():200] Probing system done +2024-12-31 21:27:31,977 DEBUG HandlerThread:1746604 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T02:27:31.971767', 'startedAt': '2025-01-01T02:27:31.561866', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3928.982, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.92, 'min': 800.0, 'max': 4700.0}, {'current': 4494.556, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4469.135, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 507.49398040771484}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 21:27:31,978 INFO HandlerThread:1746604 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 21:27:31,978 INFO HandlerThread:1746604 [system_monitor.py:probe():227] Publishing system info +2024-12-31 21:27:31,978 DEBUG HandlerThread:1746604 [system_info.py:_save_code():44] Saving code +2024-12-31 21:27:31,979 DEBUG HandlerThread:1746604 [system_info.py:_save_code():67] Saving code done +2024-12-31 21:27:31,979 DEBUG HandlerThread:1746604 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 21:27:32,042 DEBUG HandlerThread:1746604 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 21:27:32,042 INFO HandlerThread:1746604 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 21:27:32,045 DEBUG SenderThread:1746604 [sender.py:send():379] send: files +2024-12-31 21:27:32,045 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 21:27:32,045 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 21:27:32,046 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 21:27:32,277 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/s4vfpuwy-wandb-metadata.json +2024-12-31 21:27:32,282 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 21:27:32,282 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: python_packages +2024-12-31 21:27:32,284 DEBUG SenderThread:1746604 [sender.py:send():379] send: telemetry +2024-12-31 21:27:32,320 INFO wandb-upload_1:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/cwia0lzj-code/FISH/eval_robot.py +2024-12-31 21:27:32,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:32,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:27:32,325 INFO HandlerThread:1746604 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/tb" + save: true +} +control { + relay_id: "c837d0f9c03b46dc9ed679278affa699" +} +_info { + stream_id: "z0y7vp39" +} + +2024-12-31 21:27:32,325 DEBUG HandlerThread:1746604 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 21:27:32,326 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:27:32,344 INFO wandb-upload_2:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/cfcnmz7m-diff.patch +2024-12-31 21:27:32,398 DEBUG SenderThread:1746604 [sender.py:send():379] send: tbrecord +2024-12-31 21:27:32,399 DEBUG SenderThread:1746604 [sender.py:send():379] send: files +2024-12-31 21:27:32,399 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 with policy live +2024-12-31 21:27:32,579 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/yjw00hgk-events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:27:32,855 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:27:32,855 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/requirements.txt +2024-12-31 21:27:32,855 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/code/FISH/eval_robot.py +2024-12-31 21:27:32,855 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/diff.patch +2024-12-31 21:27:32,855 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:27:32,855 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-metadata.json +2024-12-31 21:27:32,855 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/code +2024-12-31 21:27:32,855 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/code/FISH +2024-12-31 21:27:33,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:34,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:34,854 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:27:35,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:36,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:37,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:27:37,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:38,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:39,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:40,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:41,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:42,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:27:42,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:43,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:44,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:45,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:46,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:47,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:27:47,283 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:27:47,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:47,434 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:27:48,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:49,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:50,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:51,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:52,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:53,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:27:53,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:54,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:55,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:56,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:57,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:58,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:27:58,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:59,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:00,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:01,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:02,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:28:02,283 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:28:02,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:03,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:03,436 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:03,861 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/config.yaml +2024-12-31 21:28:04,287 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:05,287 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:06,287 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:07,287 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:08,287 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:09,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:09,287 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:10,287 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:11,287 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:12,287 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:13,288 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:14,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:14,288 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:15,288 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:16,288 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:17,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:28:17,283 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:28:17,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:18,289 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:19,289 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:19,367 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:20,289 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:21,290 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:22,290 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:23,290 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:24,290 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:25,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:25,290 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:26,290 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:27,245 DEBUG SenderThread:1746604 [sender.py:send():379] send: config +2024-12-31 21:28:27,290 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:28,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:28,868 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:28:29,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:30,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:30,298 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:30,868 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:28:31,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:31,961 DEBUG SystemMonitor:1746604 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 21:28:31,962 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:28:32,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:28:32,283 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:28:32,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:32,869 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:28:33,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:34,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:34,870 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:28:35,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:35,789 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:36,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:36,892 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:28:36,893 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/config.yaml +2024-12-31 21:28:37,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:38,292 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:39,292 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:40,292 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:41,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:41,292 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:42,292 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:43,292 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:44,292 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:45,293 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:46,293 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:47,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:28:47,283 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:28:47,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:47,342 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:48,294 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:49,294 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:50,294 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:51,294 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:52,294 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:53,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:53,294 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:54,295 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:55,295 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:56,295 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:57,295 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:58,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:58,295 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:59,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:00,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:01,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:01,962 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:29:02,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:29:02,283 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:29:02,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:03,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:03,344 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:04,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:05,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:06,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:07,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:08,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:09,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:09,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:10,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:11,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:12,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:13,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:14,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:14,298 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:15,298 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:16,298 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:17,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:29:17,284 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:29:17,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:18,298 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:18,904 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:19,299 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:19,437 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:20,299 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:20,904 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:21,301 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:22,299 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:22,905 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:23,299 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:24,302 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:24,504 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:24,906 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:25,301 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:26,301 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:26,906 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:27,301 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:28,301 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:28,907 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:29,301 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:29,709 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:30,301 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:30,908 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:31,301 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:31,964 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:29:32,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:29:32,284 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:29:32,302 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:32,908 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:33,302 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:34,303 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:34,891 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:34,909 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:35,302 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:36,302 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:36,909 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:37,303 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:37,546 DEBUG SenderThread:1746604 [sender.py:send():379] send: files +2024-12-31 21:29:37,547 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_73394dae2cb5c0e3e76d.mp4 with policy now +2024-12-31 21:29:37,547 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:29:37,548 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:29:37,548 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:29:37,548 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:29:37,792 INFO wandb-upload_2:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/1gw6jqwc-media/videos/eval/0_eval_0_73394dae2cb5c0e3e76d.mp4 +2024-12-31 21:29:37,910 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/0_eval_0_73394dae2cb5c0e3e76d.mp4 +2024-12-31 21:29:37,910 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:29:37,910 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval +2024-12-31 21:29:37,910 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media +2024-12-31 21:29:37,910 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos +2024-12-31 21:29:38,159 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/ic7jcpja-media/videos/eval/0_eval_0_73394dae2cb5c0e3e76d.mp4 +2024-12-31 21:29:38,303 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:38,910 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:39,303 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:40,303 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:40,558 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:41,303 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:42,303 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:42,912 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:43,303 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:44,304 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:44,912 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:45,304 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:46,304 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:46,518 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:47,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:29:47,284 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:29:47,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:48,305 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:49,305 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:50,306 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:51,305 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:52,290 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:52,305 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:53,305 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:54,305 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:54,916 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:29:55,179 INFO wandb-upload_2:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/wky0b0d2-events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:29:55,306 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:56,305 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:56,916 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:57,306 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:58,241 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:58,306 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:59,306 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:00,306 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:00,918 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:01,306 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:01,964 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:30:02,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:30:02,284 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:30:02,306 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:02,918 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:03,307 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:03,417 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:04,307 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:04,919 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:05,308 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:06,307 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:06,920 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:07,308 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:08,308 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:08,637 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:08,921 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:09,308 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:10,308 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:10,921 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:11,308 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:12,316 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:12,751 DEBUG SenderThread:1746604 [sender.py:send():379] send: files +2024-12-31 21:30:12,751 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_eb820ba01fc256486a6c.mp4 with policy now +2024-12-31 21:30:12,755 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:30:12,755 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:30:12,755 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:30:12,756 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:30:12,922 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:12,922 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:30:12,922 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/1_eval_1_eb820ba01fc256486a6c.mp4 +2024-12-31 21:30:12,923 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval +2024-12-31 21:30:12,986 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/n1i42p9m-media/videos/eval/1_eval_1_eb820ba01fc256486a6c.mp4 +2024-12-31 21:30:13,238 INFO wandb-upload_2:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/jfr6ugmk-media/videos/eval/1_eval_1_eb820ba01fc256486a6c.mp4 +2024-12-31 21:30:13,311 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:13,772 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:14,310 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:14,923 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:15,314 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:16,318 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:16,923 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:17,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:30:17,291 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:30:17,333 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:18,312 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:18,862 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:19,310 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:20,313 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:20,925 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:21,313 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:22,314 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:23,313 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:24,295 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:24,320 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:25,319 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:26,315 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:27,319 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:28,317 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:29,326 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:30,295 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:30,319 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:31,319 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:31,965 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:30:32,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:30:32,285 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:30:32,318 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:32,841 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:30:32,841 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:30:32,842 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:30:32,928 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:30:32,929 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:30:33,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:34,319 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:34,929 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:35,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:35,321 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:36,322 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:36,930 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:37,345 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:38,322 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:38,931 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:39,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:40,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:40,363 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:40,932 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:41,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:42,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:42,932 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:43,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:44,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:44,933 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:45,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:45,567 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:46,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:46,934 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:47,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:30:47,285 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:30:47,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:48,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:48,934 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:49,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:50,324 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:50,784 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:50,935 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:51,333 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:51,909 DEBUG SenderThread:1746604 [sender.py:send():379] send: files +2024-12-31 21:30:51,909 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_8637db17a31d8f9e9a53.mp4 with policy now +2024-12-31 21:30:51,913 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:30:51,913 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:30:51,913 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:30:51,913 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:30:51,936 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:30:51,936 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/2_eval_2_8637db17a31d8f9e9a53.mp4 +2024-12-31 21:30:51,936 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval +2024-12-31 21:30:52,147 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/jx77f427-media/videos/eval/2_eval_2_8637db17a31d8f9e9a53.mp4 +2024-12-31 21:30:52,324 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:52,398 INFO wandb-upload_2:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/fp0x8fjc-media/videos/eval/2_eval_2_8637db17a31d8f9e9a53.mp4 +2024-12-31 21:30:52,936 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:53,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:54,330 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:55,328 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:56,189 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:56,333 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:56,937 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:57,328 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:58,327 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:58,938 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:59,334 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:00,333 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:01,331 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:01,965 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:31:01,966 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:02,289 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:31:02,289 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:31:02,344 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:03,344 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:04,339 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:05,343 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:06,342 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:07,304 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:07,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:08,343 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:09,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:09,646 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:31:09,646 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:31:09,647 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:31:09,941 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:31:09,941 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:31:10,176 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/3lslso2z-events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:31:10,343 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:10,942 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:11,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:12,343 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:12,919 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:13,343 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:14,345 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:14,943 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:15,343 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:16,343 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:16,944 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:17,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:31:17,286 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:31:17,343 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:17,935 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:18,344 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:18,944 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:19,344 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:20,345 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:20,945 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:21,345 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:22,345 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:22,946 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:23,151 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:23,345 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:24,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:24,947 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:25,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:26,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:26,947 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:27,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:28,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:28,395 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:28,948 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:29,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:30,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:30,949 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:31,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:31,966 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:31:32,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:31:32,287 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:31:32,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:32,950 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:33,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:33,646 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:34,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:34,950 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:35,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:36,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:36,951 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:37,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:38,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:38,873 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:38,952 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:39,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:40,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:40,953 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:41,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:42,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:42,953 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:43,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:44,089 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:44,348 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:44,954 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:45,348 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:46,348 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:46,955 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:47,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:31:47,286 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:31:47,348 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:48,348 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:48,956 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:49,251 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:49,348 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:50,351 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:50,422 DEBUG SenderThread:1746604 [sender.py:send():379] send: files +2024-12-31 21:31:50,422 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4f1bea0839448d87829b.mp4 with policy now +2024-12-31 21:31:50,423 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:31:50,423 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:31:50,424 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:31:50,432 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:31:50,674 INFO wandb-upload_2:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/rippbbpg-media/videos/eval/3_eval_3_4f1bea0839448d87829b.mp4 +2024-12-31 21:31:50,956 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:50,956 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:31:50,957 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/3_eval_3_4f1bea0839448d87829b.mp4 +2024-12-31 21:31:50,957 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval +2024-12-31 21:31:51,228 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/v41wqpra-media/videos/eval/3_eval_3_4f1bea0839448d87829b.mp4 +2024-12-31 21:31:51,351 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:52,354 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:52,957 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:53,361 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:54,356 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:54,958 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:55,105 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:55,358 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:56,354 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:57,354 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:58,361 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:58,959 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:59,361 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:00,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:00,365 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:01,372 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:01,966 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:32:02,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:32:02,297 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:32:02,364 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:03,367 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:04,374 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:05,374 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:05,381 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:06,372 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:07,371 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:08,382 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:09,368 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:10,376 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:11,300 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:11,377 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:12,374 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:13,380 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:14,382 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:15,382 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:16,304 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:16,387 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:17,298 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:32:17,299 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:32:17,379 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:18,390 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:19,389 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:20,387 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:21,381 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:21,440 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:22,390 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:23,385 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:24,389 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:25,391 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:26,389 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:27,302 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:27,390 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:28,390 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:29,389 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:30,393 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:31,401 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:31,966 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:32:32,298 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:32:32,298 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:32:32,365 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:32,413 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:33,414 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:34,408 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:35,412 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:36,408 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:37,417 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:38,309 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:38,420 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:39,418 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:40,418 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:41,420 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:42,423 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:43,420 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:44,307 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:44,421 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:45,436 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:46,429 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:47,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:32:47,297 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:32:47,421 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:48,427 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:49,436 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:49,437 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:50,432 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:51,435 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:52,436 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:53,434 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:54,438 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:55,310 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:55,444 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:56,439 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:57,439 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:58,457 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:59,446 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:00,449 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:01,308 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:01,456 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:01,967 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:33:02,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:33:02,296 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:33:02,456 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:03,454 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:04,457 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:05,456 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:06,369 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:06,458 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:07,459 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:08,464 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:09,460 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:10,466 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:11,465 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:12,309 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:12,477 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:13,466 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:14,471 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:15,471 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:16,491 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:17,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:33:17,296 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:33:17,408 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:17,478 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:18,476 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:19,477 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:20,491 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:21,482 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:22,479 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:23,316 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:23,489 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:24,481 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:25,483 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:26,485 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:27,489 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:28,327 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:28,493 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:29,500 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:30,489 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:31,491 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:31,967 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:33:32,299 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:33:32,299 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:33:32,494 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:33,423 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:33,501 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:34,493 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:35,495 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:36,495 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:37,503 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:38,499 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:39,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:39,503 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:40,502 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:41,500 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:42,501 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:43,505 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:43,822 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:33:43,822 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:33:43,823 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:33:43,994 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:33:43,994 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:33:44,229 INFO wandb-upload_2:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/56q81p6h-events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:33:44,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:44,878 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:44,994 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:33:45,511 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:46,505 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:46,995 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:33:47,301 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:33:47,301 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:33:47,510 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:48,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:48,995 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:33:49,511 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:50,006 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:50,504 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:50,996 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:33:51,504 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:52,504 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:52,997 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:33:53,504 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:54,504 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:54,998 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:33:55,168 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:55,505 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:56,505 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:56,998 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:33:57,549 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:58,505 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:58,999 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:33:59,505 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:00,355 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:00,505 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:01,000 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:01,505 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:01,968 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:34:02,295 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:34:02,296 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:34:02,506 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:03,001 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:03,508 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:04,506 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:05,001 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:05,506 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:05,596 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:06,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:07,002 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:07,506 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:08,506 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:09,003 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:09,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:10,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:10,812 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:11,004 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:11,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:12,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:13,005 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:13,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:14,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:15,005 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:15,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:15,973 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:16,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:17,006 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:17,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:34:17,296 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:34:17,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:18,508 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:19,007 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:19,508 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:20,508 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:21,008 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:21,151 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:21,508 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:22,508 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:23,009 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:23,508 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:24,508 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:25,009 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:25,546 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:25,689 DEBUG SenderThread:1746604 [sender.py:send():379] send: files +2024-12-31 21:34:25,689 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_4b071c579be3fd81d6bd.mp4 with policy now +2024-12-31 21:34:25,692 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:34:25,692 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:34:25,692 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:25,693 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:25,940 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/oh9x4g5d-media/videos/eval/4_eval_4_4b071c579be3fd81d6bd.mp4 +2024-12-31 21:34:26,010 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:34:26,010 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/4_eval_4_4b071c579be3fd81d6bd.mp4 +2024-12-31 21:34:26,010 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval +2024-12-31 21:34:26,269 INFO wandb-upload_2:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/8zrrmdp2-media/videos/eval/4_eval_4_4b071c579be3fd81d6bd.mp4 +2024-12-31 21:34:26,533 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:26,709 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:27,010 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:27,537 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:28,538 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:29,538 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:30,539 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:31,011 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:31,555 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:31,968 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:34:31,970 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:32,306 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:34:32,306 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:34:32,547 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:33,012 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:33,547 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:34,547 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:35,543 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:35,888 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:34:35,889 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:35,889 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:35,950 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 21:34:35,951 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 21:34:36,013 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:34:36,013 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:34:36,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2024-12-31 21:34:36,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media +2024-12-31 21:34:36,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table/eval +2024-12-31 21:34:36,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table +2024-12-31 21:34:36,166 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:34:36,220 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/kgcipoja-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2024-12-31 21:34:36,407 INFO wandb-upload_2:1746604 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpbsw0m1e3 +2024-12-31 21:34:36,500 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 21:34:36,509 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:34:36,542 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:36,787 INFO SenderThread:1746604 [sender.py:send_request_log_artifact():1525] logged artifact run-z0y7vp39-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDEzMDA4OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE2ODUyMg==', 'latestArtifact': None}} +2024-12-31 21:34:36,787 DEBUG SenderThread:1746604 [sender.py:send():379] send: files +2024-12-31 21:34:36,788 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json with policy now +2024-12-31 21:34:36,788 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:34:36,788 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,788 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,788 DEBUG SenderThread:1746604 [sender.py:send():379] send: telemetry +2024-12-31 21:34:36,789 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,789 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,789 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,789 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,790 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,790 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,790 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,792 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,792 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,792 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,792 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,793 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,793 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,793 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,793 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,794 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,794 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,794 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,794 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,794 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,795 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,795 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,795 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 21:34:37,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:37,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:34:37,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:34:37,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json +2024-12-31 21:34:37,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table/eval +2024-12-31 21:34:37,027 INFO wandb-upload_1:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/2v5ebhzu-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2024-12-31 21:34:37,184 INFO wandb-upload_2:1746604 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpbptswpsg +2024-12-31 21:34:37,249 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/c6552hiw-media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json +2024-12-31 21:34:37,556 INFO SenderThread:1746604 [sender.py:send_request_log_artifact():1525] logged artifact run-z0y7vp39-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDEzMDA5Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE2ODUyMw==', 'latestArtifact': None}} +2024-12-31 21:34:37,556 DEBUG SenderThread:1746604 [sender.py:send():379] send: files +2024-12-31 21:34:37,556 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:37,556 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json with policy now +2024-12-31 21:34:37,556 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:34:37,557 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:37,557 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:37,557 DEBUG SenderThread:1746604 [sender.py:send():379] send: exit +2024-12-31 21:34:37,557 INFO SenderThread:1746604 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 21:34:37,558 INFO SenderThread:1746604 [sender.py:send_exit():588] handling runtime: 425 +2024-12-31 21:34:37,561 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:37,561 INFO SenderThread:1746604 [sender.py:send_exit():594] send defer +2024-12-31 21:34:37,561 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:37,561 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 21:34:37,562 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:37,562 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 21:34:37,562 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 1 +2024-12-31 21:34:37,562 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:37,562 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 21:34:37,562 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:37,562 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 21:34:37,562 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 2 +2024-12-31 21:34:37,562 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:37,562 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 21:34:37,562 INFO HandlerThread:1746604 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 21:34:37,562 DEBUG SystemMonitor:1746604 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 21:34:37,562 DEBUG SystemMonitor:1746604 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 21:34:37,564 INFO HandlerThread:1746604 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 21:34:37,564 INFO HandlerThread:1746604 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 21:34:37,635 INFO HandlerThread:1746604 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 21:34:37,635 INFO HandlerThread:1746604 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 21:34:37,635 INFO HandlerThread:1746604 [interfaces.py:finish():200] Joined network monitor +2024-12-31 21:34:37,635 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:37,635 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 21:34:37,635 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 3 +2024-12-31 21:34:37,636 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:34:37,636 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:37,636 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 21:34:37,636 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:37,636 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 21:34:37,636 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 4 +2024-12-31 21:34:37,636 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:37,636 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 21:34:37,796 INFO wandb-upload_1:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/3jae89wk-media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json +2024-12-31 21:34:38,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:34:39,015 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:43,892 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:34:43,892 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:43,892 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 21:34:43,893 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:43,893 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 21:34:43,893 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 5 +2024-12-31 21:34:43,894 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:34:43,894 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:34:43,894 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:43,894 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:43,894 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 21:34:43,895 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:43,895 DEBUG SenderThread:1746604 [sender.py:send():379] send: summary +2024-12-31 21:34:43,898 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:43,899 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:43,899 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 21:34:43,899 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 6 +2024-12-31 21:34:43,899 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:43,899 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 21:34:43,899 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:43,899 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 21:34:43,903 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:44,044 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 7 +2024-12-31 21:34:44,044 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:44,044 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 21:34:44,045 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:44,045 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 21:34:44,045 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:34:44,530 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:34:44,985 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 8 +2024-12-31 21:34:44,985 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:34:44,985 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:44,985 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 21:34:44,986 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:44,986 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 21:34:44,986 INFO SenderThread:1746604 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 21:34:44,988 INFO SenderThread:1746604 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 21:34:44,990 INFO SenderThread:1746604 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 21:34:45,004 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 9 +2024-12-31 21:34:45,005 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:45,005 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 21:34:45,005 DEBUG SenderThread:1746604 [sender.py:send():379] send: artifact +2024-12-31 21:34:45,045 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:45,046 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/config.yaml +2024-12-31 21:34:45,327 INFO wandb-upload_2:1746604 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmvus7pfh +2024-12-31 21:34:45,430 INFO wandb-upload_0:1746604 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpdf8zvwtj +2024-12-31 21:34:45,504 INFO wandb-upload_1:1746604 [upload_job.py:push():88] Uploaded file /tmp/tmpgdtnwb46/wandb-job.json +2024-12-31 21:34:45,530 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:34:46,003 INFO SenderThread:1746604 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDEzMDIwNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDEyMTU3MQ==', 'versionIndex': 392}}} +2024-12-31 21:34:46,003 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:46,003 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 21:34:46,003 INFO SenderThread:1746604 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 21:34:46,046 INFO SenderThread:1746604 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files +2024-12-31 21:34:46,046 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/config.yaml config.yaml +2024-12-31 21:34:46,046 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json wandb-summary.json +2024-12-31 21:34:46,047 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:34:46,047 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/requirements.txt requirements.txt +2024-12-31 21:34:46,047 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-metadata.json wandb-metadata.json +2024-12-31 21:34:46,047 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log output.log +2024-12-31 21:34:46,047 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/diff.patch diff.patch +2024-12-31 21:34:46,051 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 21:34:46,057 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/4_eval_4_4b071c579be3fd81d6bd.mp4 media/videos/eval/4_eval_4_4b071c579be3fd81d6bd.mp4 +2024-12-31 21:34:46,057 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/2_eval_2_8637db17a31d8f9e9a53.mp4 media/videos/eval/2_eval_2_8637db17a31d8f9e9a53.mp4 +2024-12-31 21:34:46,057 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/1_eval_1_eb820ba01fc256486a6c.mp4 media/videos/eval/1_eval_1_eb820ba01fc256486a6c.mp4 +2024-12-31 21:34:46,057 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/3_eval_3_4f1bea0839448d87829b.mp4 media/videos/eval/3_eval_3_4f1bea0839448d87829b.mp4 +2024-12-31 21:34:46,057 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/0_eval_0_73394dae2cb5c0e3e76d.mp4 media/videos/eval/0_eval_0_73394dae2cb5c0e3e76d.mp4 +2024-12-31 21:34:46,057 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2024-12-31 21:34:46,057 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json +2024-12-31 21:34:46,057 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 10 +2024-12-31 21:34:46,058 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:34:46,058 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:46,058 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 21:34:46,058 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:46,058 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 21:34:46,058 INFO SenderThread:1746604 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:34:46,227 INFO wandb-upload_2:1746604 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/config.yaml +2024-12-31 21:34:46,231 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:34:46,284 INFO wandb-upload_3:1746604 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/requirements.txt +2024-12-31 21:34:46,293 INFO wandb-upload_4:1746604 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:46,320 INFO wandb-upload_1:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/qtrx7o4i-events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:34:46,521 INFO Thread-11 (_thread_body):1746604 [sender.py:transition_state():614] send defer: 11 +2024-12-31 21:34:46,521 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:46,521 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 21:34:46,521 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:46,521 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 21:34:46,521 INFO SenderThread:1746604 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 21:34:46,523 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 12 +2024-12-31 21:34:46,523 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:46,523 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 21:34:46,523 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:46,523 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 21:34:46,523 INFO SenderThread:1746604 [file_stream.py:finish():601] file stream finish called +2024-12-31 21:34:46,531 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:34:46,825 INFO SenderThread:1746604 [file_stream.py:finish():605] file stream finish is done +2024-12-31 21:34:46,825 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 13 +2024-12-31 21:34:46,825 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:34:46,826 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:46,826 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 21:34:46,826 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:46,826 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 21:34:46,826 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 14 +2024-12-31 21:34:46,826 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:46,826 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 21:34:46,826 DEBUG SenderThread:1746604 [sender.py:send():379] send: final +2024-12-31 21:34:46,827 DEBUG SenderThread:1746604 [sender.py:send():379] send: footer +2024-12-31 21:34:46,827 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:46,827 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 21:34:46,827 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:34:46,827 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:34:46,827 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 21:34:46,828 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:34:46,828 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 21:34:46,829 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 21:34:46,829 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: server_info +2024-12-31 21:34:46,831 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:46,884 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:34:46,884 INFO MainThread:1746604 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 21:34:46,884 INFO MainThread:1746604 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 21:34:46,885 INFO MainThread:1746604 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 21:34:46,885 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 21:34:46,885 INFO HandlerThread:1746604 [handler.py:finish():882] shutting down handler +2024-12-31 21:34:47,829 INFO WriterThread:1746604 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/run-z0y7vp39.wandb +2024-12-31 21:34:47,884 INFO SenderThread:1746604 [sender.py:finish():1615] shutting down sender +2024-12-31 21:34:47,884 INFO SenderThread:1746604 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:34:47,884 INFO SenderThread:1746604 [file_pusher.py:join():175] waiting for file pusher diff --git a/z0y7vp39/wandb/debug.log b/z0y7vp39/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..920d32c02dfeb11e187639cd0e347df86a1d5d0d --- /dev/null +++ b/z0y7vp39/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-31 21:27:31,564 INFO MainThread:1746382 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 21:27:31,565 INFO MainThread:1746382 [wandb_setup.py:_flush():76] Configure stats pid to 1746382 +2024-12-31 21:27:31,565 INFO MainThread:1746382 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 21:27:31,565 INFO MainThread:1746382 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/settings +2024-12-31 21:27:31,565 INFO MainThread:1746382 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 21:27:31,565 INFO MainThread:1746382 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 21:27:31,565 INFO MainThread:1746382 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 21:27:31,565 INFO MainThread:1746382 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 21:27:31,566 INFO MainThread:1746382 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/logs/debug.log +2024-12-31 21:27:31,566 INFO MainThread:1746382 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/logs/debug-internal.log +2024-12-31 21:27:31,566 INFO MainThread:1746382 [wandb_init.py:init():569] calling init triggers +2024-12-31 21:27:31,566 INFO MainThread:1746382 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39'} +2024-12-31 21:27:31,567 INFO MainThread:1746382 [wandb_init.py:init():619] starting backend +2024-12-31 21:27:31,567 INFO MainThread:1746382 [wandb_init.py:init():623] setting up manager +2024-12-31 21:27:31,570 INFO MainThread:1746382 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 21:27:31,572 INFO MainThread:1746382 [wandb_init.py:init():631] backend started and connected +2024-12-31 21:27:31,583 INFO MainThread:1746382 [wandb_init.py:init():720] updated telemetry +2024-12-31 21:27:31,590 INFO MainThread:1746382 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 21:27:31,869 INFO MainThread:1746382 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 21:27:31,906 INFO MainThread:1746382 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 21:27:31,907 INFO MainThread:1746382 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 21:27:32,283 INFO MainThread:1746382 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 21:27:32,283 INFO MainThread:1746382 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 21:27:32,283 INFO MainThread:1746382 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 21:27:32,283 INFO MainThread:1746382 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 21:27:32,285 INFO MainThread:1746382 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 21:27:32,285 INFO MainThread:1746382 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/tb, True +2024-12-31 21:28:27,244 INFO MainThread:1746382 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2024-12-31 21:34:47,885 WARNING MsgRouterThr:1746382 [router.py:message_loop():77] message_loop has been closed diff --git a/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/code/FISH/eval_robot.py b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..41791869dc3657b820d72a52a39ee14580eb36bd --- /dev/null +++ b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + "config.grasped_obj_name": grasped_obj_name, + "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + total_num_episodes += filtered_run.summary_metrics['episode'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/config.yaml b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05a34bc56a3f6ecea2fa3eb86084bd3b0c428f19 --- /dev/null +++ b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1006_0' +wandb_notes: + desc: null + value: 1006_0_req_1352_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735698451 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/diff.patch b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..e09588e1c3357af1242f50ab4ab556d71e342fdb --- /dev/null +++ b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/diff.patch @@ -0,0 +1,33 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..c76321b 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,7 +133,10 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1007_0' ++# wandb_run_id: '1007_0' ++ ++# RGBD+mask+contact(all ftrs)+act history ++wandb_run_id: '1006_0' + + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..fe71904 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,11 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ '1006_0', + ] + + checkpoint_epoch = 12000 diff --git a/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json new file mode 100644 index 0000000000000000000000000000000000000000..cc7e5e91447669fe989ab08b6e5b18934b0ea7ca --- /dev/null +++ b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.4, 0.04400911936343024, 0.8621948956718848, 0.05324718365067565, 0.803534984527916, 0.11712844677476308, 0.7487166672447589, 0.23282015328982517, 0.6637700199753331]]} \ No newline at end of file diff --git a/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json new file mode 100644 index 0000000000000000000000000000000000000000..1d95a7bddc9f4697c0a8c2264aab5142c2121875 --- /dev/null +++ b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[2, 5, 0.4, 0.04070010790474144, 0.8728086947649158, 0.09036684044447546, 0.8503108768954606, 0.12382420905007341, 0.7396230101106143, 0.1259237081560607, 0.7419417798035252]]} \ No newline at end of file diff --git a/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/0_eval_0_73394dae2cb5c0e3e76d.mp4 b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/0_eval_0_73394dae2cb5c0e3e76d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3ac1f78da1799e0324a44a95a1f046f40978735c --- /dev/null +++ b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/0_eval_0_73394dae2cb5c0e3e76d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73394dae2cb5c0e3e76ddd01183d5fb1de9dd95392128b29d1c428faa8a3c842 +size 706666 diff --git a/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/1_eval_1_eb820ba01fc256486a6c.mp4 b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/1_eval_1_eb820ba01fc256486a6c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2bb2219d7538095fdfc950d07325cca3dbe2161c --- /dev/null +++ b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/1_eval_1_eb820ba01fc256486a6c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb820ba01fc256486a6ce64bb8e09317279dd63b1bb04cdd34a8f99ac85e8747 +size 621660 diff --git a/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/2_eval_2_8637db17a31d8f9e9a53.mp4 b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/2_eval_2_8637db17a31d8f9e9a53.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..125c3ea8f907fb60bc6b352247b3a9d23947e9c4 --- /dev/null +++ b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/2_eval_2_8637db17a31d8f9e9a53.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8637db17a31d8f9e9a53089018168b3d8b4ac0608ff98f962c66d4250bf80a3f +size 689867 diff --git a/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/3_eval_3_4f1bea0839448d87829b.mp4 b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/3_eval_3_4f1bea0839448d87829b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c5f3f33e763488c891a5d8d3f36a663bfbbaf4e5 --- /dev/null +++ b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/3_eval_3_4f1bea0839448d87829b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f1bea0839448d87829bcde0ef5cbba8ece5cbb3d99968a605f93e85d36e44ce +size 1566098 diff --git a/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/4_eval_4_4b071c579be3fd81d6bd.mp4 b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/4_eval_4_4b071c579be3fd81d6bd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6fb95bc1f9658ed466df014cc7f06502d24c4e5f --- /dev/null +++ b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/4_eval_4_4b071c579be3fd81d6bd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b071c579be3fd81d6bdd5c557c3c3e485a6e0d9f3e8e02f49c17a533ec3a014 +size 1534428 diff --git a/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..adeaef9021f8c916b37569bab8b72bc1e6d81012 --- /dev/null +++ b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log @@ -0,0 +1,4128 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735698507.245082]: resetting environment +[INFO] [1735698507.253293]: cleared current plan +[INFO] [1735698507.254198]: moving to home +[INFO] [1735698508.256958]: reached home +[INFO] [1735698508.258380]: reset action history +[INFO] [1735698510.967807]: environment reset +Starting episode 0 +[INFO] [1735698510.971674]: resetting environment +[INFO] [1735698510.976399]: cleared current plan +[INFO] [1735698510.977511]: moving to home +[INFO] [1735698511.979597]: reached home +[INFO] [1735698511.981811]: reset action history +[INFO] [1735698514.741452]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735698557.387980]: resetting environment +[INFO] [1735698557.392742]: cleared current plan +[INFO] [1735698557.393679]: moving to home +[INFO] [1735698558.395888]: reached home +[INFO] [1735698558.397256]: reset action history +[INFO] [1735698561.107492]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/episode_rosbags/episode_0_2024-12-31-21-29-21.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1746382.log +started roslaunch server http://158.130.50.37:38381/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1746718] +process[print_text-2]: started with pid [1746742] +process[rosbag_record-3]: started with pid [1746743] +started bagging! +[INFO] [1735698561.876985]: received mask_obs with header seq: 1799 +stamp: + secs: 1735698561 + nsecs: 879040718 +frame_id: '' and grasped_object_mask header seq: 176001 +stamp: + secs: 1735698561 + nsecs: 656332731 +frame_id: "camera_color_optical_frame" +[INFO] [1735698562.391500]: received mask_obs with header seq: 1800 +stamp: + secs: 1735698562 + nsecs: 395065307 +frame_id: '' and grasped_object_mask header seq: 176017 +stamp: + secs: 1735698562 + nsecs: 188572168 +frame_id: "camera_color_optical_frame" +[INFO] [1735698562.978763]: received mask_obs with header seq: 1801 +stamp: + secs: 1735698562 + nsecs: 981482744 +frame_id: '' and grasped_object_mask header seq: 176034 +stamp: + secs: 1735698562 + nsecs: 754009008 +frame_id: "camera_color_optical_frame" +[INFO] [1735698563.227314]: received mask_obs with header seq: 1802 +stamp: + secs: 1735698563 + nsecs: 230981588 +frame_id: '' and grasped_object_mask header seq: 176042 +stamp: + secs: 1735698563 + nsecs: 20143747 +frame_id: "camera_color_optical_frame" +[INFO] [1735698563.497380]: received mask_obs with header seq: 1803 +stamp: + secs: 1735698563 + nsecs: 500728368 +frame_id: '' and grasped_object_mask header seq: 176049 +stamp: + secs: 1735698563 + nsecs: 252993584 +frame_id: "camera_color_optical_frame" +[INFO] [1735698563.740637]: received mask_obs with header seq: 1804 +stamp: + secs: 1735698563 + nsecs: 743887662 +frame_id: '' and grasped_object_mask header seq: 176057 +stamp: + secs: 1735698563 + nsecs: 519089460 +frame_id: "camera_color_optical_frame" +[INFO] [1735698563.991700]: received mask_obs with header seq: 1805 +stamp: + secs: 1735698563 + nsecs: 995362758 +frame_id: '' and grasped_object_mask header seq: 176064 +stamp: + secs: 1735698563 + nsecs: 751914740 +frame_id: "camera_color_optical_frame" +[INFO] [1735698564.246263]: received mask_obs with header seq: 1806 +stamp: + secs: 1735698564 + nsecs: 249561071 +frame_id: '' and grasped_object_mask header seq: 176072 +stamp: + secs: 1735698564 + nsecs: 18065214 +frame_id: "camera_color_optical_frame" +[INFO] [1735698564.504019]: received mask_obs with header seq: 1807 +stamp: + secs: 1735698564 + nsecs: 507693290 +frame_id: '' and grasped_object_mask header seq: 176079 +stamp: + secs: 1735698564 + nsecs: 250928402 +frame_id: "camera_color_optical_frame" +[INFO] [1735698564.754872]: received mask_obs with header seq: 1808 +stamp: + secs: 1735698564 + nsecs: 758464574 +frame_id: '' and grasped_object_mask header seq: 176087 +stamp: + secs: 1735698564 + nsecs: 517040253 +frame_id: "camera_color_optical_frame" +[INFO] [1735698565.012807]: received mask_obs with header seq: 1809 +stamp: + secs: 1735698565 + nsecs: 16363859 +frame_id: '' and grasped_object_mask header seq: 176095 +stamp: + secs: 1735698564 + nsecs: 783145905 +frame_id: "camera_color_optical_frame" +[INFO] [1735698565.281857]: received mask_obs with header seq: 1810 +stamp: + secs: 1735698565 + nsecs: 285509824 +frame_id: '' and grasped_object_mask header seq: 176103 +stamp: + secs: 1735698565 + nsecs: 49284458 +frame_id: "camera_color_optical_frame" +[INFO] [1735698565.544175]: received mask_obs with header seq: 1811 +stamp: + secs: 1735698565 + nsecs: 547675371 +frame_id: '' and grasped_object_mask header seq: 176111 +stamp: + secs: 1735698565 + nsecs: 315401554 +frame_id: "camera_color_optical_frame" +[INFO] [1735698565.805151]: received mask_obs with header seq: 1812 +stamp: + secs: 1735698565 + nsecs: 808459043 +frame_id: '' and grasped_object_mask header seq: 176119 +stamp: + secs: 1735698565 + nsecs: 581497908 +frame_id: "camera_color_optical_frame" +[INFO] [1735698566.060825]: received mask_obs with header seq: 1813 +stamp: + secs: 1735698566 + nsecs: 64355373 +frame_id: '' and grasped_object_mask header seq: 176126 +stamp: + secs: 1735698565 + nsecs: 814346075 +frame_id: "camera_color_optical_frame" +[INFO] [1735698566.315291]: received mask_obs with header seq: 1814 +stamp: + secs: 1735698566 + nsecs: 318873882 +frame_id: '' and grasped_object_mask header seq: 176134 +stamp: + secs: 1735698566 + nsecs: 80492496 +frame_id: "camera_color_optical_frame" +[INFO] [1735698566.578452]: received mask_obs with header seq: 1815 +stamp: + secs: 1735698566 + nsecs: 582154273 +frame_id: '' and grasped_object_mask header seq: 176142 +stamp: + secs: 1735698566 + nsecs: 346618176 +frame_id: "camera_color_optical_frame" +[INFO] [1735698566.840291]: received mask_obs with header seq: 1816 +stamp: + secs: 1735698566 + nsecs: 843689441 +frame_id: '' and grasped_object_mask header seq: 176150 +stamp: + secs: 1735698566 + nsecs: 612724781 +frame_id: "camera_color_optical_frame" +[INFO] [1735698567.099956]: received mask_obs with header seq: 1817 +stamp: + secs: 1735698567 + nsecs: 103234767 +frame_id: '' and grasped_object_mask header seq: 176157 +stamp: + secs: 1735698566 + nsecs: 845580816 +frame_id: "camera_color_optical_frame" +[INFO] [1735698567.352887]: received mask_obs with header seq: 1818 +stamp: + secs: 1735698567 + nsecs: 356350183 +frame_id: '' and grasped_object_mask header seq: 176165 +stamp: + secs: 1735698567 + nsecs: 111713886 +frame_id: "camera_color_optical_frame" +[INFO] [1735698567.616255]: received mask_obs with header seq: 1819 +stamp: + secs: 1735698567 + nsecs: 619837999 +frame_id: '' and grasped_object_mask header seq: 176173 +stamp: + secs: 1735698567 + nsecs: 377824545 +frame_id: "camera_color_optical_frame" +[INFO] [1735698567.874939]: received mask_obs with header seq: 1820 +stamp: + secs: 1735698567 + nsecs: 878630638 +frame_id: '' and grasped_object_mask header seq: 176181 +stamp: + secs: 1735698567 + nsecs: 643913746 +frame_id: "camera_color_optical_frame" +[INFO] [1735698568.139919]: received mask_obs with header seq: 1821 +stamp: + secs: 1735698568 + nsecs: 143536329 +frame_id: '' and grasped_object_mask header seq: 176189 +stamp: + secs: 1735698567 + nsecs: 910051584 +frame_id: "camera_color_optical_frame" +[INFO] [1735698568.390886]: received mask_obs with header seq: 1822 +stamp: + secs: 1735698568 + nsecs: 394253253 +frame_id: '' and grasped_object_mask header seq: 176196 +stamp: + secs: 1735698568 + nsecs: 142924309 +frame_id: "camera_color_optical_frame" +[INFO] [1735698568.650257]: received mask_obs with header seq: 1823 +stamp: + secs: 1735698568 + nsecs: 653899192 +frame_id: '' and grasped_object_mask header seq: 176204 +stamp: + secs: 1735698568 + nsecs: 409044981 +frame_id: "camera_color_optical_frame" +[INFO] [1735698568.908678]: received mask_obs with header seq: 1824 +stamp: + secs: 1735698568 + nsecs: 912237882 +frame_id: '' and grasped_object_mask header seq: 176212 +stamp: + secs: 1735698568 + nsecs: 675144672 +frame_id: "camera_color_optical_frame" +[INFO] [1735698569.179440]: received mask_obs with header seq: 1825 +stamp: + secs: 1735698569 + nsecs: 182897806 +frame_id: '' and grasped_object_mask header seq: 176220 +stamp: + secs: 1735698568 + nsecs: 941276789 +frame_id: "camera_color_optical_frame" +[INFO] [1735698569.441243]: received mask_obs with header seq: 1826 +stamp: + secs: 1735698569 + nsecs: 444775342 +frame_id: '' and grasped_object_mask header seq: 176228 +stamp: + secs: 1735698569 + nsecs: 207404375 +frame_id: "camera_color_optical_frame" +[INFO] [1735698569.708569]: received mask_obs with header seq: 1827 +stamp: + secs: 1735698569 + nsecs: 712132930 +frame_id: '' and grasped_object_mask header seq: 176236 +stamp: + secs: 1735698569 + nsecs: 473510742 +frame_id: "camera_color_optical_frame" +[INFO] [1735698569.960713]: received mask_obs with header seq: 1828 +stamp: + secs: 1735698569 + nsecs: 964296102 +frame_id: '' and grasped_object_mask header seq: 176243 +stamp: + secs: 1735698569 + nsecs: 706337214 +frame_id: "camera_color_optical_frame" +[INFO] [1735698570.205816]: received mask_obs with header seq: 1829 +stamp: + secs: 1735698570 + nsecs: 209235668 +frame_id: '' and grasped_object_mask header seq: 176251 +stamp: + secs: 1735698569 + nsecs: 974009752 +frame_id: "camera_color_optical_frame" +[INFO] [1735698570.455571]: received mask_obs with header seq: 1830 +stamp: + secs: 1735698570 + nsecs: 459190130 +frame_id: '' and grasped_object_mask header seq: 176258 +stamp: + secs: 1735698570 + nsecs: 208544493 +frame_id: "camera_color_optical_frame" +[INFO] [1735698570.717464]: received mask_obs with header seq: 1831 +stamp: + secs: 1735698570 + nsecs: 720704793 +frame_id: '' and grasped_object_mask header seq: 176266 +stamp: + secs: 1735698570 + nsecs: 476731539 +frame_id: "camera_color_optical_frame" +[INFO] [1735698570.975148]: received mask_obs with header seq: 1832 +stamp: + secs: 1735698570 + nsecs: 978828430 +frame_id: '' and grasped_object_mask header seq: 176274 +stamp: + secs: 1735698570 + nsecs: 745076418 +frame_id: "camera_color_optical_frame" +[INFO] [1735698571.241832]: received mask_obs with header seq: 1833 +stamp: + secs: 1735698571 + nsecs: 245438814 +frame_id: '' and grasped_object_mask header seq: 176282 +stamp: + secs: 1735698571 + nsecs: 11564970 +frame_id: "camera_color_optical_frame" +[INFO] [1735698571.496532]: received mask_obs with header seq: 1834 +stamp: + secs: 1735698571 + nsecs: 500036239 +frame_id: '' and grasped_object_mask header seq: 176289 +stamp: + secs: 1735698571 + nsecs: 244711876 +frame_id: "camera_color_optical_frame" +[INFO] [1735698571.752041]: received mask_obs with header seq: 1835 +stamp: + secs: 1735698571 + nsecs: 755403280 +frame_id: '' and grasped_object_mask header seq: 176297 +stamp: + secs: 1735698571 + nsecs: 511165142 +frame_id: "camera_color_optical_frame" +[INFO] [1735698572.020377]: received mask_obs with header seq: 1836 +stamp: + secs: 1735698572 + nsecs: 23808479 +frame_id: '' and grasped_object_mask header seq: 176305 +stamp: + secs: 1735698571 + nsecs: 777618408 +frame_id: "camera_color_optical_frame" +[INFO] [1735698572.277944]: received mask_obs with header seq: 1837 +stamp: + secs: 1735698572 + nsecs: 281489372 +frame_id: '' and grasped_object_mask header seq: 176313 +stamp: + secs: 1735698572 + nsecs: 44073582 +frame_id: "camera_color_optical_frame" +[INFO] [1735698572.544743]: received mask_obs with header seq: 1838 +stamp: + secs: 1735698572 + nsecs: 548241138 +frame_id: '' and grasped_object_mask header seq: 176321 +stamp: + secs: 1735698572 + nsecs: 310527563 +frame_id: "camera_color_optical_frame" +[INFO] [1735698572.792679]: received mask_obs with header seq: 1839 +stamp: + secs: 1735698572 + nsecs: 796205997 +frame_id: '' and grasped_object_mask header seq: 176328 +stamp: + secs: 1735698572 + nsecs: 543675423 +frame_id: "camera_color_optical_frame" +[INFO] [1735698573.052795]: received mask_obs with header seq: 1840 +stamp: + secs: 1735698573 + nsecs: 56457757 +frame_id: '' and grasped_object_mask header seq: 176336 +stamp: + secs: 1735698572 + nsecs: 810129404 +frame_id: "camera_color_optical_frame" +[INFO] [1735698573.319719]: received mask_obs with header seq: 1841 +stamp: + secs: 1735698573 + nsecs: 323181390 +frame_id: '' and grasped_object_mask header seq: 176344 +stamp: + secs: 1735698573 + nsecs: 76580524 +frame_id: "camera_color_optical_frame" +[INFO] [1735698573.579759]: received mask_obs with header seq: 1842 +stamp: + secs: 1735698573 + nsecs: 583400011 +frame_id: '' and grasped_object_mask header seq: 176352 +stamp: + secs: 1735698573 + nsecs: 343031645 +frame_id: "camera_color_optical_frame" +[INFO] [1735698573.842273]: received mask_obs with header seq: 1843 +stamp: + secs: 1735698573 + nsecs: 845692157 +frame_id: '' and grasped_object_mask header seq: 176360 +stamp: + secs: 1735698573 + nsecs: 609483004 +frame_id: "camera_color_optical_frame" +[INFO] [1735698574.119745]: received mask_obs with header seq: 1844 +stamp: + secs: 1735698574 + nsecs: 123246192 +frame_id: '' and grasped_object_mask header seq: 176368 +stamp: + secs: 1735698573 + nsecs: 875934124 +frame_id: "camera_color_optical_frame" +[INFO] [1735698574.376201]: received mask_obs with header seq: 1845 +stamp: + secs: 1735698574 + nsecs: 379780292 +frame_id: '' and grasped_object_mask header seq: 176376 +stamp: + secs: 1735698574 + nsecs: 142383099 +frame_id: "camera_color_optical_frame" +[INFO] [1735698574.642540]: received mask_obs with header seq: 1846 +stamp: + secs: 1735698574 + nsecs: 646099567 +frame_id: '' and grasped_object_mask header seq: 176384 +stamp: + secs: 1735698574 + nsecs: 408832073 +frame_id: "camera_color_optical_frame" +[INFO] [1735698574.890798]: received mask_obs with header seq: 1847 +stamp: + secs: 1735698574 + nsecs: 894337654 +frame_id: '' and grasped_object_mask header seq: 176391 +stamp: + secs: 1735698574 + nsecs: 641975403 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 111602844 for nearest: 1735698575020027221 - target: 1735698574908424377 at idx 181 +[INFO] [1735698575.153047]: received mask_obs with header seq: 1848 +stamp: + secs: 1735698575 + nsecs: 156688213 +frame_id: '' and grasped_object_mask header seq: 176399 +stamp: + secs: 1735698574 + nsecs: 908424377 +frame_id: "camera_color_optical_frame" +[INFO] [1735698575.409037]: received mask_obs with header seq: 1849 +stamp: + secs: 1735698575 + nsecs: 412626504 +frame_id: '' and grasped_object_mask header seq: 176407 +stamp: + secs: 1735698575 + nsecs: 174873352 +frame_id: "camera_color_optical_frame" +[INFO] [1735698575.674089]: received mask_obs with header seq: 1850 +stamp: + secs: 1735698575 + nsecs: 677486896 +frame_id: '' and grasped_object_mask header seq: 176415 +stamp: + secs: 1735698575 + nsecs: 441322327 +frame_id: "camera_color_optical_frame" +[INFO] [1735698575.939273]: received mask_obs with header seq: 1851 +stamp: + secs: 1735698575 + nsecs: 942916393 +frame_id: '' and grasped_object_mask header seq: 176423 +stamp: + secs: 1735698575 + nsecs: 707772255 +frame_id: "camera_color_optical_frame" +[INFO] [1735698576.192969]: received mask_obs with header seq: 1852 +stamp: + secs: 1735698576 + nsecs: 196504354 +frame_id: '' and grasped_object_mask header seq: 176430 +stamp: + secs: 1735698575 + nsecs: 940914392 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 112687841 for nearest: 1735698576320050493 - target: 1735698576207362652 at idx 198 +[INFO] [1735698576.449785]: received mask_obs with header seq: 1853 +stamp: + secs: 1735698576 + nsecs: 453402280 +frame_id: '' and grasped_object_mask header seq: 176438 +stamp: + secs: 1735698576 + nsecs: 207362652 +frame_id: "camera_color_optical_frame" +preempting episode +[INFO] [1735698576.549864]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735698576.704350]: received mask_obs with header seq: 1854 +stamp: + secs: 1735698576 + nsecs: 707827568 +frame_id: '' and grasped_object_mask header seq: 176446 +stamp: + secs: 1735698576 + nsecs: 473812103 +frame_id: "camera_color_optical_frame" +[INFO] [1735698577.320885]: Stored episode 1. +[INFO] [1735698577.321271]: Saving video... +[INFO] [1735698577.535013]: Video saved! +Starting episode 1 +[INFO] [1735698577.550812]: resetting environment +[INFO] [1735698577.557646]: cleared current plan +[INFO] [1735698577.557897]: moving to home +[INFO] [1735698580.759379]: reached home +[INFO] [1735698580.759977]: reset action history +[INFO] [1735698583.469978]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735698594.513054]: resetting environment +[INFO] [1735698594.524468]: cleared current plan +[INFO] [1735698594.526642]: moving to home +[INFO] [1735698595.529291]: reached home +[INFO] [1735698595.529741]: reset action history +[INFO] [1735698598.241195]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/episode_rosbags/episode_1_2024-12-31-21-29-58.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1746382.log +started roslaunch server http://158.130.50.37:44495/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1746877] +process[print_text-5]: started with pid [1746878] +process[rosbag_record-6]: started with pid [1746902] +started bagging! +[INFO] [1735698598.997403]: received mask_obs with header seq: 1855 +stamp: + secs: 1735698599 + nsecs: 851392 +frame_id: '' and grasped_object_mask header seq: 177115 +stamp: + secs: 1735698598 + nsecs: 749712706 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 102842133 for nearest: 1735698599120028298 - target: 1735698599017186165 at idx 15 +[INFO] [1735698599.242819]: received mask_obs with header seq: 1856 +stamp: + secs: 1735698599 + nsecs: 245359659 +frame_id: '' and grasped_object_mask header seq: 177123 +stamp: + secs: 1735698599 + nsecs: 17186165 +frame_id: "camera_color_optical_frame" +[INFO] [1735698599.486617]: received mask_obs with header seq: 1857 +stamp: + secs: 1735698599 + nsecs: 489934921 +frame_id: '' and grasped_object_mask header seq: 177130 +stamp: + secs: 1735698599 + nsecs: 251677513 +frame_id: "camera_color_optical_frame" +[INFO] [1735698599.726571]: received mask_obs with header seq: 1858 +stamp: + secs: 1735698599 + nsecs: 730190277 +frame_id: '' and grasped_object_mask header seq: 177138 +stamp: + secs: 1735698599 + nsecs: 519811869 +frame_id: "camera_color_optical_frame" +[INFO] [1735698599.966711]: received mask_obs with header seq: 1859 +stamp: + secs: 1735698599 + nsecs: 970338821 +frame_id: '' and grasped_object_mask header seq: 177145 +stamp: + secs: 1735698599 + nsecs: 754556417 +frame_id: "camera_color_optical_frame" +[INFO] [1735698600.209866]: received mask_obs with header seq: 1860 +stamp: + secs: 1735698600 + nsecs: 213341236 +frame_id: '' and grasped_object_mask header seq: 177152 +stamp: + secs: 1735698599 + nsecs: 988122225 +frame_id: "camera_color_optical_frame" +[INFO] [1735698600.466486]: received mask_obs with header seq: 1861 +stamp: + secs: 1735698600 + nsecs: 470000743 +frame_id: '' and grasped_object_mask header seq: 177159 +stamp: + secs: 1735698600 + nsecs: 221271038 +frame_id: "camera_color_optical_frame" +[INFO] [1735698600.728386]: received mask_obs with header seq: 1862 +stamp: + secs: 1735698600 + nsecs: 731823444 +frame_id: '' and grasped_object_mask header seq: 177167 +stamp: + secs: 1735698600 + nsecs: 487729073 +frame_id: "camera_color_optical_frame" +[INFO] [1735698601.020629]: received mask_obs with header seq: 1863 +stamp: + secs: 1735698601 + nsecs: 24298667 +frame_id: '' and grasped_object_mask header seq: 177175 +stamp: + secs: 1735698600 + nsecs: 754186630 +frame_id: "camera_color_optical_frame" +[INFO] [1735698601.278285]: received mask_obs with header seq: 1864 +stamp: + secs: 1735698601 + nsecs: 281678676 +frame_id: '' and grasped_object_mask header seq: 177184 +stamp: + secs: 1735698601 + nsecs: 53952456 +frame_id: "camera_color_optical_frame" +[INFO] [1735698601.532168]: received mask_obs with header seq: 1865 +stamp: + secs: 1735698601 + nsecs: 535544157 +frame_id: '' and grasped_object_mask header seq: 177191 +stamp: + secs: 1735698601 + nsecs: 287102461 +frame_id: "camera_color_optical_frame" +[INFO] [1735698601.807070]: received mask_obs with header seq: 1866 +stamp: + secs: 1735698601 + nsecs: 810120344 +frame_id: '' and grasped_object_mask header seq: 177199 +stamp: + secs: 1735698601 + nsecs: 553561687 +frame_id: "camera_color_optical_frame" +[INFO] [1735698602.061518]: received mask_obs with header seq: 1867 +stamp: + secs: 1735698602 + nsecs: 65000295 +frame_id: '' and grasped_object_mask header seq: 177207 +stamp: + secs: 1735698601 + nsecs: 820019722 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 100245452 for nearest: 1735698602220024085 - target: 1735698602119778633 at idx 63 +[INFO] [1735698602.368507]: received mask_obs with header seq: 1868 +stamp: + secs: 1735698602 + nsecs: 370671987 +frame_id: '' and grasped_object_mask header seq: 177216 +stamp: + secs: 1735698602 + nsecs: 119778633 +frame_id: "camera_color_optical_frame" +[INFO] [1735698602.617366]: received mask_obs with header seq: 1869 +stamp: + secs: 1735698602 + nsecs: 620915651 +frame_id: '' and grasped_object_mask header seq: 177224 +stamp: + secs: 1735698602 + nsecs: 386232376 +frame_id: "camera_color_optical_frame" +[INFO] [1735698602.879315]: received mask_obs with header seq: 1870 +stamp: + secs: 1735698602 + nsecs: 882359027 +frame_id: '' and grasped_object_mask header seq: 177232 +stamp: + secs: 1735698602 + nsecs: 652685165 +frame_id: "camera_color_optical_frame" +[INFO] [1735698603.150049]: received mask_obs with header seq: 1871 +stamp: + secs: 1735698603 + nsecs: 153685808 +frame_id: '' and grasped_object_mask header seq: 177240 +stamp: + secs: 1735698602 + nsecs: 919140577 +frame_id: "camera_color_optical_frame" +[INFO] [1735698603.416594]: received mask_obs with header seq: 1872 +stamp: + secs: 1735698603 + nsecs: 419990777 +frame_id: '' and grasped_object_mask header seq: 177248 +stamp: + secs: 1735698603 + nsecs: 185600519 +frame_id: "camera_color_optical_frame" +[INFO] [1735698603.686974]: received mask_obs with header seq: 1873 +stamp: + secs: 1735698603 + nsecs: 690624475 +frame_id: '' and grasped_object_mask header seq: 177256 +stamp: + secs: 1735698603 + nsecs: 452062130 +frame_id: "camera_color_optical_frame" +[INFO] [1735698603.953860]: received mask_obs with header seq: 1874 +stamp: + secs: 1735698603 + nsecs: 957242012 +frame_id: '' and grasped_object_mask header seq: 177264 +stamp: + secs: 1735698603 + nsecs: 718522787 +frame_id: "camera_color_optical_frame" +[INFO] [1735698604.230237]: received mask_obs with header seq: 1875 +stamp: + secs: 1735698604 + nsecs: 233812332 +frame_id: '' and grasped_object_mask header seq: 177272 +stamp: + secs: 1735698603 + nsecs: 984974623 +frame_id: "camera_color_optical_frame" +[INFO] [1735698604.482006]: received mask_obs with header seq: 1876 +stamp: + secs: 1735698604 + nsecs: 485616683 +frame_id: '' and grasped_object_mask header seq: 177280 +stamp: + secs: 1735698604 + nsecs: 251420736 +frame_id: "camera_color_optical_frame" +[INFO] [1735698604.743173]: received mask_obs with header seq: 1877 +stamp: + secs: 1735698604 + nsecs: 746690273 +frame_id: '' and grasped_object_mask header seq: 177288 +stamp: + secs: 1735698604 + nsecs: 517866373 +frame_id: "camera_color_optical_frame" +[INFO] [1735698605.005009]: received mask_obs with header seq: 1878 +stamp: + secs: 1735698605 + nsecs: 8557081 +frame_id: '' and grasped_object_mask header seq: 177296 +stamp: + secs: 1735698604 + nsecs: 784312963 +frame_id: "camera_color_optical_frame" +[INFO] [1735698605.257697]: received mask_obs with header seq: 1879 +stamp: + secs: 1735698605 + nsecs: 261324882 +frame_id: '' and grasped_object_mask header seq: 177303 +stamp: + secs: 1735698605 + nsecs: 17454147 +frame_id: "camera_color_optical_frame" +[INFO] [1735698605.514484]: received mask_obs with header seq: 1880 +stamp: + secs: 1735698605 + nsecs: 518097400 +frame_id: '' and grasped_object_mask header seq: 177311 +stamp: + secs: 1735698605 + nsecs: 283903122 +frame_id: "camera_color_optical_frame" +[INFO] [1735698605.775521]: received mask_obs with header seq: 1881 +stamp: + secs: 1735698605 + nsecs: 778808355 +frame_id: '' and grasped_object_mask header seq: 177319 +stamp: + secs: 1735698605 + nsecs: 550350904 +frame_id: "camera_color_optical_frame" +[INFO] [1735698606.043378]: received mask_obs with header seq: 1882 +stamp: + secs: 1735698606 + nsecs: 46950340 +frame_id: '' and grasped_object_mask header seq: 177327 +stamp: + secs: 1735698605 + nsecs: 816798925 +frame_id: "camera_color_optical_frame" +[INFO] [1735698606.295566]: received mask_obs with header seq: 1883 +stamp: + secs: 1735698606 + nsecs: 299174070 +frame_id: '' and grasped_object_mask header seq: 177334 +stamp: + secs: 1735698606 + nsecs: 49941778 +frame_id: "camera_color_optical_frame" +[INFO] [1735698606.555048]: received mask_obs with header seq: 1884 +stamp: + secs: 1735698606 + nsecs: 558622360 +frame_id: '' and grasped_object_mask header seq: 177342 +stamp: + secs: 1735698606 + nsecs: 316390753 +frame_id: "camera_color_optical_frame" +[INFO] [1735698606.813260]: received mask_obs with header seq: 1885 +stamp: + secs: 1735698606 + nsecs: 816819906 +frame_id: '' and grasped_object_mask header seq: 177350 +stamp: + secs: 1735698606 + nsecs: 582840204 +frame_id: "camera_color_optical_frame" +[INFO] [1735698607.077874]: received mask_obs with header seq: 1886 +stamp: + secs: 1735698607 + nsecs: 79602718 +frame_id: '' and grasped_object_mask header seq: 177358 +stamp: + secs: 1735698606 + nsecs: 849287987 +frame_id: "camera_color_optical_frame" +[INFO] [1735698607.339698]: received mask_obs with header seq: 1887 +stamp: + secs: 1735698607 + nsecs: 343136310 +frame_id: '' and grasped_object_mask header seq: 177366 +stamp: + secs: 1735698607 + nsecs: 115728855 +frame_id: "camera_color_optical_frame" +[INFO] [1735698607.594550]: received mask_obs with header seq: 1888 +stamp: + secs: 1735698607 + nsecs: 598123788 +frame_id: '' and grasped_object_mask header seq: 177373 +stamp: + secs: 1735698607 + nsecs: 348862648 +frame_id: "camera_color_optical_frame" +[INFO] [1735698607.851388]: received mask_obs with header seq: 1889 +stamp: + secs: 1735698607 + nsecs: 854996681 +frame_id: '' and grasped_object_mask header seq: 177381 +stamp: + secs: 1735698607 + nsecs: 615301132 +frame_id: "camera_color_optical_frame" +[INFO] [1735698608.110350]: received mask_obs with header seq: 1890 +stamp: + secs: 1735698608 + nsecs: 113454818 +frame_id: '' and grasped_object_mask header seq: 177389 +stamp: + secs: 1735698607 + nsecs: 881741524 +frame_id: "camera_color_optical_frame" +[INFO] [1735698608.372418]: received mask_obs with header seq: 1891 +stamp: + secs: 1735698608 + nsecs: 375513553 +frame_id: '' and grasped_object_mask header seq: 177397 +stamp: + secs: 1735698608 + nsecs: 148184061 +frame_id: "camera_color_optical_frame" +[INFO] [1735698608.636654]: received mask_obs with header seq: 1892 +stamp: + secs: 1735698608 + nsecs: 639974832 +frame_id: '' and grasped_object_mask header seq: 177405 +stamp: + secs: 1735698608 + nsecs: 414626360 +frame_id: "camera_color_optical_frame" +[INFO] [1735698608.908752]: received mask_obs with header seq: 1893 +stamp: + secs: 1735698608 + nsecs: 912260770 +frame_id: '' and grasped_object_mask header seq: 177413 +stamp: + secs: 1735698608 + nsecs: 681068420 +frame_id: "camera_color_optical_frame" +[INFO] [1735698609.172750]: received mask_obs with header seq: 1894 +stamp: + secs: 1735698609 + nsecs: 176174163 +frame_id: '' and grasped_object_mask header seq: 177421 +stamp: + secs: 1735698608 + nsecs: 947512627 +frame_id: "camera_color_optical_frame" +[INFO] [1735698609.435281]: received mask_obs with header seq: 1895 +stamp: + secs: 1735698609 + nsecs: 438864231 +frame_id: '' and grasped_object_mask header seq: 177429 +stamp: + secs: 1735698609 + nsecs: 213957310 +frame_id: "camera_color_optical_frame" +[INFO] [1735698609.685589]: received mask_obs with header seq: 1896 +stamp: + secs: 1735698609 + nsecs: 689202785 +frame_id: '' and grasped_object_mask header seq: 177436 +stamp: + secs: 1735698609 + nsecs: 447095394 +frame_id: "camera_color_optical_frame" +[INFO] [1735698609.943461]: received mask_obs with header seq: 1897 +stamp: + secs: 1735698609 + nsecs: 946968555 +frame_id: '' and grasped_object_mask header seq: 177444 +stamp: + secs: 1735698609 + nsecs: 713540554 +frame_id: "camera_color_optical_frame" +[INFO] [1735698610.203510]: received mask_obs with header seq: 1898 +stamp: + secs: 1735698610 + nsecs: 206909418 +frame_id: '' and grasped_object_mask header seq: 177452 +stamp: + secs: 1735698609 + nsecs: 979991436 +frame_id: "camera_color_optical_frame" +[INFO] [1735698610.467847]: received mask_obs with header seq: 1899 +stamp: + secs: 1735698610 + nsecs: 471411943 +frame_id: '' and grasped_object_mask header seq: 177460 +stamp: + secs: 1735698610 + nsecs: 246447086 +frame_id: "camera_color_optical_frame" +[INFO] [1735698610.718769]: received mask_obs with header seq: 1900 +stamp: + secs: 1735698610 + nsecs: 722021341 +frame_id: '' and grasped_object_mask header seq: 177467 +stamp: + secs: 1735698610 + nsecs: 479596376 +frame_id: "camera_color_optical_frame" +[INFO] [1735698610.983447]: received mask_obs with header seq: 1901 +stamp: + secs: 1735698610 + nsecs: 987107038 +frame_id: '' and grasped_object_mask header seq: 177475 +stamp: + secs: 1735698610 + nsecs: 746052504 +frame_id: "camera_color_optical_frame" +[INFO] [1735698611.249899]: received mask_obs with header seq: 1902 +stamp: + secs: 1735698611 + nsecs: 253500699 +frame_id: '' and grasped_object_mask header seq: 177483 +stamp: + secs: 1735698611 + nsecs: 12501240 +frame_id: "camera_color_optical_frame" +[INFO] [1735698611.509984]: received mask_obs with header seq: 1903 +stamp: + secs: 1735698611 + nsecs: 513596534 +frame_id: '' and grasped_object_mask header seq: 177491 +stamp: + secs: 1735698611 + nsecs: 278947592 +frame_id: "camera_color_optical_frame" +preempting episode +[INFO] [1735698611.668999]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735698611.775040]: received mask_obs with header seq: 1904 +stamp: + secs: 1735698611 + nsecs: 772865772 +frame_id: '' and grasped_object_mask header seq: 177499 +stamp: + secs: 1735698611 + nsecs: 545394659 +frame_id: "camera_color_optical_frame" +[INFO] [1735698612.482273]: Stored episode 2. +[INFO] [1735698612.488958]: Saving video... +[INFO] [1735698612.724285]: Video saved! +Starting episode 2 +[INFO] [1735698612.757482]: resetting environment +[INFO] [1735698612.772177]: cleared current plan +[INFO] [1735698612.772397]: moving to home +[INFO] [1735698616.077225]: reached home +[INFO] [1735698616.080615]: reset action history +[INFO] [1735698618.861912]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735698632.243877]: resetting environment +[INFO] [1735698632.253632]: cleared current plan +[INFO] [1735698632.253887]: moving to home +[INFO] [1735698633.255595]: reached home +[INFO] [1735698633.256884]: reset action history +[INFO] [1735698635.989165]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/episode_rosbags/episode_2_2024-12-31-21-30-36.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1746382.log +started roslaunch server http://158.130.50.37:35655/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1747043] +process[print_text-8]: started with pid [1747044] +process[rosbag_record-9]: started with pid [1747045] +started bagging! +[INFO] [1735698636.827133]: received mask_obs with header seq: 1905 +stamp: + secs: 1735698636 + nsecs: 830513477 +frame_id: '' and grasped_object_mask header seq: 178251 +stamp: + secs: 1735698636 + nsecs: 591853857 +frame_id: "camera_color_optical_frame" +[INFO] [1735698637.070334]: received mask_obs with header seq: 1906 +stamp: + secs: 1735698637 + nsecs: 73915958 +frame_id: '' and grasped_object_mask header seq: 178259 +stamp: + secs: 1735698636 + nsecs: 858301401 +frame_id: "camera_color_optical_frame" +[INFO] [1735698637.303672]: received mask_obs with header seq: 1907 +stamp: + secs: 1735698637 + nsecs: 307363510 +frame_id: '' and grasped_object_mask header seq: 178266 +stamp: + secs: 1735698637 + nsecs: 91455936 +frame_id: "camera_color_optical_frame" +[INFO] [1735698637.547858]: received mask_obs with header seq: 1908 +stamp: + secs: 1735698637 + nsecs: 551495075 +frame_id: '' and grasped_object_mask header seq: 178273 +stamp: + secs: 1735698637 + nsecs: 324611187 +frame_id: "camera_color_optical_frame" +[INFO] [1735698637.791927]: received mask_obs with header seq: 1909 +stamp: + secs: 1735698637 + nsecs: 795339345 +frame_id: '' and grasped_object_mask header seq: 178280 +stamp: + secs: 1735698637 + nsecs: 557765245 +frame_id: "camera_color_optical_frame" +[INFO] [1735698638.039151]: received mask_obs with header seq: 1910 +stamp: + secs: 1735698638 + nsecs: 41766405 +frame_id: '' and grasped_object_mask header seq: 178287 +stamp: + secs: 1735698637 + nsecs: 790920734 +frame_id: "camera_color_optical_frame" +[INFO] [1735698638.286386]: received mask_obs with header seq: 1911 +stamp: + secs: 1735698638 + nsecs: 289175510 +frame_id: '' and grasped_object_mask header seq: 178295 +stamp: + secs: 1735698638 + nsecs: 57379484 +frame_id: "camera_color_optical_frame" +[INFO] [1735698638.549807]: received mask_obs with header seq: 1912 +stamp: + secs: 1735698638 + nsecs: 553162336 +frame_id: '' and grasped_object_mask header seq: 178302 +stamp: + secs: 1735698638 + nsecs: 290530682 +frame_id: "camera_color_optical_frame" +[INFO] [1735698638.806241]: received mask_obs with header seq: 1913 +stamp: + secs: 1735698638 + nsecs: 809829235 +frame_id: '' and grasped_object_mask header seq: 178310 +stamp: + secs: 1735698638 + nsecs: 556989193 +frame_id: "camera_color_optical_frame" +[INFO] [1735698639.067632]: received mask_obs with header seq: 1914 +stamp: + secs: 1735698639 + nsecs: 71273565 +frame_id: '' and grasped_object_mask header seq: 178318 +stamp: + secs: 1735698638 + nsecs: 823448181 +frame_id: "camera_color_optical_frame" +[INFO] [1735698639.321847]: received mask_obs with header seq: 1915 +stamp: + secs: 1735698639 + nsecs: 325538635 +frame_id: '' and grasped_object_mask header seq: 178326 +stamp: + secs: 1735698639 + nsecs: 89900017 +frame_id: "camera_color_optical_frame" +[INFO] [1735698639.587151]: received mask_obs with header seq: 1916 +stamp: + secs: 1735698639 + nsecs: 590791225 +frame_id: '' and grasped_object_mask header seq: 178334 +stamp: + secs: 1735698639 + nsecs: 356352091 +frame_id: "camera_color_optical_frame" +[INFO] [1735698639.850775]: received mask_obs with header seq: 1917 +stamp: + secs: 1735698639 + nsecs: 854447841 +frame_id: '' and grasped_object_mask header seq: 178341 +stamp: + secs: 1735698639 + nsecs: 589496374 +frame_id: "camera_color_optical_frame" +[INFO] [1735698640.107518]: received mask_obs with header seq: 1918 +stamp: + secs: 1735698640 + nsecs: 110963106 +frame_id: '' and grasped_object_mask header seq: 178349 +stamp: + secs: 1735698639 + nsecs: 855949402 +frame_id: "camera_color_optical_frame" +[INFO] [1735698640.363197]: received mask_obs with header seq: 1919 +stamp: + secs: 1735698640 + nsecs: 366694450 +frame_id: '' and grasped_object_mask header seq: 178357 +stamp: + secs: 1735698640 + nsecs: 122408867 +frame_id: "camera_color_optical_frame" +[INFO] [1735698640.623111]: received mask_obs with header seq: 1920 +stamp: + secs: 1735698640 + nsecs: 626394510 +frame_id: '' and grasped_object_mask header seq: 178365 +stamp: + secs: 1735698640 + nsecs: 388869047 +frame_id: "camera_color_optical_frame" +[INFO] [1735698640.887010]: received mask_obs with header seq: 1921 +stamp: + secs: 1735698640 + nsecs: 890250921 +frame_id: '' and grasped_object_mask header seq: 178373 +stamp: + secs: 1735698640 + nsecs: 655329227 +frame_id: "camera_color_optical_frame" +[INFO] [1735698641.142445]: received mask_obs with header seq: 1922 +stamp: + secs: 1735698641 + nsecs: 145879983 +frame_id: '' and grasped_object_mask header seq: 178380 +stamp: + secs: 1735698640 + nsecs: 888481855 +frame_id: "camera_color_optical_frame" +[INFO] [1735698641.400886]: received mask_obs with header seq: 1923 +stamp: + secs: 1735698641 + nsecs: 404201030 +frame_id: '' and grasped_object_mask header seq: 178388 +stamp: + secs: 1735698641 + nsecs: 154936314 +frame_id: "camera_color_optical_frame" +[INFO] [1735698641.658914]: received mask_obs with header seq: 1924 +stamp: + secs: 1735698641 + nsecs: 662459373 +frame_id: '' and grasped_object_mask header seq: 178396 +stamp: + secs: 1735698641 + nsecs: 421390533 +frame_id: "camera_color_optical_frame" +[INFO] [1735698641.924032]: received mask_obs with header seq: 1925 +stamp: + secs: 1735698641 + nsecs: 927453041 +frame_id: '' and grasped_object_mask header seq: 178404 +stamp: + secs: 1735698641 + nsecs: 687845945 +frame_id: "camera_color_optical_frame" +[INFO] [1735698642.193382]: received mask_obs with header seq: 1926 +stamp: + secs: 1735698642 + nsecs: 196979999 +frame_id: '' and grasped_object_mask header seq: 178411 +stamp: + secs: 1735698641 + nsecs: 920991421 +frame_id: "camera_color_optical_frame" +[INFO] [1735698642.454915]: received mask_obs with header seq: 1927 +stamp: + secs: 1735698642 + nsecs: 458241701 +frame_id: '' and grasped_object_mask header seq: 178420 +stamp: + secs: 1735698642 + nsecs: 220748425 +frame_id: "camera_color_optical_frame" +[INFO] [1735698642.718981]: received mask_obs with header seq: 1928 +stamp: + secs: 1735698642 + nsecs: 721868753 +frame_id: '' and grasped_object_mask header seq: 178428 +stamp: + secs: 1735698642 + nsecs: 487198830 +frame_id: "camera_color_optical_frame" +[INFO] [1735698642.971016]: received mask_obs with header seq: 1929 +stamp: + secs: 1735698642 + nsecs: 974639177 +frame_id: '' and grasped_object_mask header seq: 178435 +stamp: + secs: 1735698642 + nsecs: 720343351 +frame_id: "camera_color_optical_frame" +[INFO] [1735698643.227562]: received mask_obs with header seq: 1930 +stamp: + secs: 1735698643 + nsecs: 231244325 +frame_id: '' and grasped_object_mask header seq: 178443 +stamp: + secs: 1735698642 + nsecs: 986793041 +frame_id: "camera_color_optical_frame" +[INFO] [1735698643.488419]: received mask_obs with header seq: 1931 +stamp: + secs: 1735698643 + nsecs: 492059230 +frame_id: '' and grasped_object_mask header seq: 178451 +stamp: + secs: 1735698643 + nsecs: 253242970 +frame_id: "camera_color_optical_frame" +[INFO] [1735698643.754078]: received mask_obs with header seq: 1932 +stamp: + secs: 1735698643 + nsecs: 757738590 +frame_id: '' and grasped_object_mask header seq: 178459 +stamp: + secs: 1735698643 + nsecs: 519693375 +frame_id: "camera_color_optical_frame" +[INFO] [1735698644.018664]: received mask_obs with header seq: 1933 +stamp: + secs: 1735698644 + nsecs: 22178411 +frame_id: '' and grasped_object_mask header seq: 178466 +stamp: + secs: 1735698643 + nsecs: 752836466 +frame_id: "camera_color_optical_frame" +[INFO] [1735698644.262951]: received mask_obs with header seq: 1934 +stamp: + secs: 1735698644 + nsecs: 266474008 +frame_id: '' and grasped_object_mask header seq: 178474 +stamp: + secs: 1735698644 + nsecs: 19281864 +frame_id: "camera_color_optical_frame" +[INFO] [1735698644.525737]: received mask_obs with header seq: 1935 +stamp: + secs: 1735698644 + nsecs: 529366493 +frame_id: '' and grasped_object_mask header seq: 178482 +stamp: + secs: 1735698644 + nsecs: 285726070 +frame_id: "camera_color_optical_frame" +[INFO] [1735698644.787873]: received mask_obs with header seq: 1936 +stamp: + secs: 1735698644 + nsecs: 791508913 +frame_id: '' and grasped_object_mask header seq: 178490 +stamp: + secs: 1735698644 + nsecs: 552168608 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -165223561 for nearest: 1735698644653387346 - target: 1735698644818610907 at idx 118 +[INFO] [1735698645.053521]: received mask_obs with header seq: 1937 +stamp: + secs: 1735698645 + nsecs: 57148456 +frame_id: '' and grasped_object_mask header seq: 178498 +stamp: + secs: 1735698644 + nsecs: 818610907 +frame_id: "camera_color_optical_frame" +[INFO] [1735698645.304217]: received mask_obs with header seq: 1938 +stamp: + secs: 1735698645 + nsecs: 307827949 +frame_id: '' and grasped_object_mask header seq: 178505 +stamp: + secs: 1735698645 + nsecs: 51749229 +frame_id: "camera_color_optical_frame" +[INFO] [1735698645.566942]: received mask_obs with header seq: 1939 +stamp: + secs: 1735698645 + nsecs: 570296287 +frame_id: '' and grasped_object_mask header seq: 178513 +stamp: + secs: 1735698645 + nsecs: 318192720 +frame_id: "camera_color_optical_frame" +[INFO] [1735698645.825446]: received mask_obs with header seq: 1940 +stamp: + secs: 1735698645 + nsecs: 829032897 +frame_id: '' and grasped_object_mask header seq: 178521 +stamp: + secs: 1735698645 + nsecs: 584635735 +frame_id: "camera_color_optical_frame" +[INFO] [1735698646.093798]: received mask_obs with header seq: 1941 +stamp: + secs: 1735698646 + nsecs: 97418069 +frame_id: '' and grasped_object_mask header seq: 178529 +stamp: + secs: 1735698645 + nsecs: 851078987 +frame_id: "camera_color_optical_frame" +[INFO] [1735698646.366947]: received mask_obs with header seq: 1942 +stamp: + secs: 1735698646 + nsecs: 370595216 +frame_id: '' and grasped_object_mask header seq: 178537 +stamp: + secs: 1735698646 + nsecs: 117538691 +frame_id: "camera_color_optical_frame" +[INFO] [1735698646.620367]: received mask_obs with header seq: 1943 +stamp: + secs: 1735698646 + nsecs: 623831272 +frame_id: '' and grasped_object_mask header seq: 178545 +stamp: + secs: 1735698646 + nsecs: 383997202 +frame_id: "camera_color_optical_frame" +[INFO] [1735698646.873163]: received mask_obs with header seq: 1944 +stamp: + secs: 1735698646 + nsecs: 876717090 +frame_id: '' and grasped_object_mask header seq: 178552 +stamp: + secs: 1735698646 + nsecs: 617149591 +frame_id: "camera_color_optical_frame" +[INFO] [1735698647.133891]: received mask_obs with header seq: 1945 +stamp: + secs: 1735698647 + nsecs: 137337923 +frame_id: '' and grasped_object_mask header seq: 178560 +stamp: + secs: 1735698646 + nsecs: 883607626 +frame_id: "camera_color_optical_frame" +[INFO] [1735698647.388170]: received mask_obs with header seq: 1946 +stamp: + secs: 1735698647 + nsecs: 391819715 +frame_id: '' and grasped_object_mask header seq: 178568 +stamp: + secs: 1735698647 + nsecs: 150058508 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 103516265 for nearest: 1735698647520025893 - target: 1735698647416509628 at idx 152 +[INFO] [1735698647.650527]: received mask_obs with header seq: 1947 +stamp: + secs: 1735698647 + nsecs: 654085636 +frame_id: '' and grasped_object_mask header seq: 178576 +stamp: + secs: 1735698647 + nsecs: 416509628 +frame_id: "camera_color_optical_frame" +[INFO] [1735698647.911594]: received mask_obs with header seq: 1948 +stamp: + secs: 1735698647 + nsecs: 914756774 +frame_id: '' and grasped_object_mask header seq: 178584 +stamp: + secs: 1735698647 + nsecs: 682960272 +frame_id: "camera_color_optical_frame" +[INFO] [1735698648.178004]: received mask_obs with header seq: 1949 +stamp: + secs: 1735698648 + nsecs: 181205987 +frame_id: '' and grasped_object_mask header seq: 178592 +stamp: + secs: 1735698647 + nsecs: 949413061 +frame_id: "camera_color_optical_frame" +[INFO] [1735698648.439651]: received mask_obs with header seq: 1950 +stamp: + secs: 1735698648 + nsecs: 442343950 +frame_id: '' and grasped_object_mask header seq: 178600 +stamp: + secs: 1735698648 + nsecs: 215865850 +frame_id: "camera_color_optical_frame" +[INFO] [1735698648.703070]: received mask_obs with header seq: 1951 +stamp: + secs: 1735698648 + nsecs: 706526041 +frame_id: '' and grasped_object_mask header seq: 178608 +stamp: + secs: 1735698648 + nsecs: 482319117 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 104563068 for nearest: 1735698648820028852 - target: 1735698648715465784 at idx 167 +[INFO] [1735698648.972255]: received mask_obs with header seq: 1952 +stamp: + secs: 1735698648 + nsecs: 975897312 +frame_id: '' and grasped_object_mask header seq: 178615 +stamp: + secs: 1735698648 + nsecs: 715465784 +frame_id: "camera_color_optical_frame" +[INFO] [1735698649.238969]: received mask_obs with header seq: 1953 +stamp: + secs: 1735698649 + nsecs: 241105079 +frame_id: '' and grasped_object_mask header seq: 178624 +stamp: + secs: 1735698649 + nsecs: 15225649 +frame_id: "camera_color_optical_frame" +[INFO] [1735698649.494074]: received mask_obs with header seq: 1954 +stamp: + secs: 1735698649 + nsecs: 497386932 +frame_id: '' and grasped_object_mask header seq: 178631 +stamp: + secs: 1735698649 + nsecs: 248371124 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 105201694 for nearest: 1735698649620025131 - target: 1735698649514823437 at idx 178 +[INFO] [1735698649.750183]: received mask_obs with header seq: 1955 +stamp: + secs: 1735698649 + nsecs: 753558635 +frame_id: '' and grasped_object_mask header seq: 178639 +stamp: + secs: 1735698649 + nsecs: 514823437 +frame_id: "camera_color_optical_frame" +[INFO] [1735698650.007803]: received mask_obs with header seq: 1956 +stamp: + secs: 1735698650 + nsecs: 10679960 +frame_id: '' and grasped_object_mask header seq: 178647 +stamp: + secs: 1735698649 + nsecs: 781276941 +frame_id: "camera_color_optical_frame" +[INFO] [1735698650.273969]: received mask_obs with header seq: 1957 +stamp: + secs: 1735698650 + nsecs: 276407241 +frame_id: '' and grasped_object_mask header seq: 178655 +stamp: + secs: 1735698650 + nsecs: 47731638 +frame_id: "camera_color_optical_frame" +[INFO] [1735698650.530550]: received mask_obs with header seq: 1958 +stamp: + secs: 1735698650 + nsecs: 533593654 +frame_id: '' and grasped_object_mask header seq: 178663 +stamp: + secs: 1735698650 + nsecs: 314187765 +frame_id: "camera_color_optical_frame" +[INFO] [1735698650.784066]: received mask_obs with header seq: 1959 +stamp: + secs: 1735698650 + nsecs: 787691831 +frame_id: '' and grasped_object_mask header seq: 178670 +stamp: + secs: 1735698650 + nsecs: 547337055 +frame_id: "camera_color_optical_frame" +preempting episode +[INFO] [1735698650.882361]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1735698651.042913]: received mask_obs with header seq: 1960 +stamp: + secs: 1735698651 + nsecs: 44257879 +frame_id: '' and grasped_object_mask header seq: 178678 +stamp: + secs: 1735698650 + nsecs: 813793421 +frame_id: "camera_color_optical_frame" +[INFO] [1735698651.640004]: Stored episode 3. +[INFO] [1735698651.645426]: Saving video... +[INFO] [1735698651.882547]: Video saved! +Starting episode 3 +[INFO] [1735698651.915943]: resetting environment +[INFO] [1735698651.942954]: cleared current plan +[INFO] [1735698651.943164]: moving to home +[INFO] [1735698655.148491]: reached home +[INFO] [1735698655.151289]: reset action history +[INFO] [1735698658.069731]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735698669.094038]: resetting environment +[INFO] [1735698669.124619]: cleared current plan +[INFO] [1735698669.134982]: moving to home +[INFO] [1735698670.137210]: reached home +[INFO] [1735698670.141528]: reset action history +[INFO] [1735698672.918651]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/episode_rosbags/episode_3_2024-12-31-21-31-12.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1746382.log +started roslaunch server http://158.130.50.37:44633/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1747184] +process[print_text-11]: started with pid [1747185] +process[rosbag_record-12]: started with pid [1747186] +started bagging! +[INFO] [1735698675.177501]: received mask_obs with header seq: 1961 +stamp: + secs: 1735698675 + nsecs: 178097486 +frame_id: '' and grasped_object_mask header seq: 179403 +stamp: + secs: 1735698674 + nsecs: 964368105 +frame_id: "camera_color_optical_frame" +[INFO] [1735698675.404237]: received mask_obs with header seq: 1962 +stamp: + secs: 1735698675 + nsecs: 407625436 +frame_id: '' and grasped_object_mask header seq: 179410 +stamp: + secs: 1735698675 + nsecs: 197258472 +frame_id: "camera_color_optical_frame" +[INFO] [1735698675.643449]: received mask_obs with header seq: 1963 +stamp: + secs: 1735698675 + nsecs: 645829916 +frame_id: '' and grasped_object_mask header seq: 179417 +stamp: + secs: 1735698675 + nsecs: 430132866 +frame_id: "camera_color_optical_frame" +[INFO] [1735698675.896255]: received mask_obs with header seq: 1964 +stamp: + secs: 1735698675 + nsecs: 899699687 +frame_id: '' and grasped_object_mask header seq: 179424 +stamp: + secs: 1735698675 + nsecs: 662992716 +frame_id: "camera_color_optical_frame" +[INFO] [1735698676.143060]: received mask_obs with header seq: 1965 +stamp: + secs: 1735698676 + nsecs: 146626472 +frame_id: '' and grasped_object_mask header seq: 179432 +stamp: + secs: 1735698675 + nsecs: 929107189 +frame_id: "camera_color_optical_frame" +[INFO] [1735698676.394153]: received mask_obs with header seq: 1966 +stamp: + secs: 1735698676 + nsecs: 396921634 +frame_id: '' and grasped_object_mask header seq: 179439 +stamp: + secs: 1735698676 + nsecs: 161989450 +frame_id: "camera_color_optical_frame" +[INFO] [1735698676.643586]: received mask_obs with header seq: 1967 +stamp: + secs: 1735698676 + nsecs: 646764993 +frame_id: '' and grasped_object_mask header seq: 179447 +stamp: + secs: 1735698676 + nsecs: 428121090 +frame_id: "camera_color_optical_frame" +[INFO] [1735698676.903843]: received mask_obs with header seq: 1968 +stamp: + secs: 1735698676 + nsecs: 907415390 +frame_id: '' and grasped_object_mask header seq: 179454 +stamp: + secs: 1735698676 + nsecs: 660969973 +frame_id: "camera_color_optical_frame" +[INFO] [1735698677.180732]: received mask_obs with header seq: 1969 +stamp: + secs: 1735698677 + nsecs: 184396266 +frame_id: '' and grasped_object_mask header seq: 179462 +stamp: + secs: 1735698676 + nsecs: 927073479 +frame_id: "camera_color_optical_frame" +[INFO] [1735698677.420986]: received mask_obs with header seq: 1970 +stamp: + secs: 1735698677 + nsecs: 424485683 +frame_id: '' and grasped_object_mask header seq: 179470 +stamp: + secs: 1735698677 + nsecs: 193230152 +frame_id: "camera_color_optical_frame" +[INFO] [1735698677.683671]: received mask_obs with header seq: 1971 +stamp: + secs: 1735698677 + nsecs: 687306404 +frame_id: '' and grasped_object_mask header seq: 179478 +stamp: + secs: 1735698677 + nsecs: 459367752 +frame_id: "camera_color_optical_frame" +[INFO] [1735698677.934489]: received mask_obs with header seq: 1972 +stamp: + secs: 1735698677 + nsecs: 938171863 +frame_id: '' and grasped_object_mask header seq: 179485 +stamp: + secs: 1735698677 + nsecs: 692220211 +frame_id: "camera_color_optical_frame" +[INFO] [1735698678.195408]: received mask_obs with header seq: 1973 +stamp: + secs: 1735698678 + nsecs: 199062585 +frame_id: '' and grasped_object_mask header seq: 179493 +stamp: + secs: 1735698677 + nsecs: 958335400 +frame_id: "camera_color_optical_frame" +[INFO] [1735698678.452305]: received mask_obs with header seq: 1974 +stamp: + secs: 1735698678 + nsecs: 455461740 +frame_id: '' and grasped_object_mask header seq: 179501 +stamp: + secs: 1735698678 + nsecs: 224481106 +frame_id: "camera_color_optical_frame" +[INFO] [1735698678.717209]: received mask_obs with header seq: 1975 +stamp: + secs: 1735698678 + nsecs: 719964742 +frame_id: '' and grasped_object_mask header seq: 179509 +stamp: + secs: 1735698678 + nsecs: 490604877 +frame_id: "camera_color_optical_frame" +[INFO] [1735698678.969762]: received mask_obs with header seq: 1976 +stamp: + secs: 1735698678 + nsecs: 973179578 +frame_id: '' and grasped_object_mask header seq: 179516 +stamp: + secs: 1735698678 + nsecs: 723446846 +frame_id: "camera_color_optical_frame" +[INFO] [1735698679.217120]: received mask_obs with header seq: 1977 +stamp: + secs: 1735698679 + nsecs: 220750808 +frame_id: '' and grasped_object_mask header seq: 179524 +stamp: + secs: 1735698678 + nsecs: 989561796 +frame_id: "camera_color_optical_frame" +[INFO] [1735698679.480289]: received mask_obs with header seq: 1978 +stamp: + secs: 1735698679 + nsecs: 483951568 +frame_id: '' and grasped_object_mask header seq: 179532 +stamp: + secs: 1735698679 + nsecs: 255706072 +frame_id: "camera_color_optical_frame" +[INFO] [1735698679.730591]: received mask_obs with header seq: 1979 +stamp: + secs: 1735698679 + nsecs: 734211444 +frame_id: '' and grasped_object_mask header seq: 179539 +stamp: + secs: 1735698679 + nsecs: 488566160 +frame_id: "camera_color_optical_frame" +[INFO] [1735698679.989627]: received mask_obs with header seq: 1980 +stamp: + secs: 1735698679 + nsecs: 993153333 +frame_id: '' and grasped_object_mask header seq: 179547 +stamp: + secs: 1735698679 + nsecs: 754672050 +frame_id: "camera_color_optical_frame" +[INFO] [1735698680.248555]: received mask_obs with header seq: 1981 +stamp: + secs: 1735698680 + nsecs: 252119779 +frame_id: '' and grasped_object_mask header seq: 179555 +stamp: + secs: 1735698680 + nsecs: 20798683 +frame_id: "camera_color_optical_frame" +[INFO] [1735698680.518264]: received mask_obs with header seq: 1982 +stamp: + secs: 1735698680 + nsecs: 521501541 +frame_id: '' and grasped_object_mask header seq: 179563 +stamp: + secs: 1735698680 + nsecs: 286945581 +frame_id: "camera_color_optical_frame" +[INFO] [1735698680.782510]: received mask_obs with header seq: 1983 +stamp: + secs: 1735698680 + nsecs: 786009311 +frame_id: '' and grasped_object_mask header seq: 179570 +stamp: + secs: 1735698680 + nsecs: 519807577 +frame_id: "camera_color_optical_frame" +[INFO] [1735698681.052350]: received mask_obs with header seq: 1984 +stamp: + secs: 1735698681 + nsecs: 54950952 +frame_id: '' and grasped_object_mask header seq: 179578 +stamp: + secs: 1735698680 + nsecs: 785916090 +frame_id: "camera_color_optical_frame" +[INFO] [1735698681.315835]: received mask_obs with header seq: 1985 +stamp: + secs: 1735698681 + nsecs: 319401025 +frame_id: '' and grasped_object_mask header seq: 179587 +stamp: + secs: 1735698681 + nsecs: 85312605 +frame_id: "camera_color_optical_frame" +[INFO] [1735698681.585293]: received mask_obs with header seq: 1986 +stamp: + secs: 1735698681 + nsecs: 588453531 +frame_id: '' and grasped_object_mask header seq: 179595 +stamp: + secs: 1735698681 + nsecs: 351441383 +frame_id: "camera_color_optical_frame" +[INFO] [1735698681.850066]: received mask_obs with header seq: 1987 +stamp: + secs: 1735698681 + nsecs: 853502035 +frame_id: '' and grasped_object_mask header seq: 179603 +stamp: + secs: 1735698681 + nsecs: 617548943 +frame_id: "camera_color_optical_frame" +[INFO] [1735698682.118718]: received mask_obs with header seq: 1988 +stamp: + secs: 1735698682 + nsecs: 121796846 +frame_id: '' and grasped_object_mask header seq: 179611 +stamp: + secs: 1735698681 + nsecs: 883636236 +frame_id: "camera_color_optical_frame" +[INFO] [1735698682.375468]: received mask_obs with header seq: 1989 +stamp: + secs: 1735698682 + nsecs: 379100084 +frame_id: '' and grasped_object_mask header seq: 179618 +stamp: + secs: 1735698682 + nsecs: 116527081 +frame_id: "camera_color_optical_frame" +[INFO] [1735698682.628505]: received mask_obs with header seq: 1990 +stamp: + secs: 1735698682 + nsecs: 631862878 +frame_id: '' and grasped_object_mask header seq: 179626 +stamp: + secs: 1735698682 + nsecs: 382670879 +frame_id: "camera_color_optical_frame" +[INFO] [1735698682.890637]: received mask_obs with header seq: 1991 +stamp: + secs: 1735698682 + nsecs: 894300937 +frame_id: '' and grasped_object_mask header seq: 179634 +stamp: + secs: 1735698682 + nsecs: 648792982 +frame_id: "camera_color_optical_frame" +[INFO] [1735698683.151175]: received mask_obs with header seq: 1992 +stamp: + secs: 1735698683 + nsecs: 154547214 +frame_id: '' and grasped_object_mask header seq: 179642 +stamp: + secs: 1735698682 + nsecs: 915113688 +frame_id: "camera_color_optical_frame" +[INFO] [1735698683.424473]: received mask_obs with header seq: 1993 +stamp: + secs: 1735698683 + nsecs: 427781820 +frame_id: '' and grasped_object_mask header seq: 179650 +stamp: + secs: 1735698683 + nsecs: 183044910 +frame_id: "camera_color_optical_frame" +[INFO] [1735698683.687186]: received mask_obs with header seq: 1994 +stamp: + secs: 1735698683 + nsecs: 690201759 +frame_id: '' and grasped_object_mask header seq: 179658 +stamp: + secs: 1735698683 + nsecs: 451132774 +frame_id: "camera_color_optical_frame" +[INFO] [1735698683.951250]: received mask_obs with header seq: 1995 +stamp: + secs: 1735698683 + nsecs: 954561948 +frame_id: '' and grasped_object_mask header seq: 179666 +stamp: + secs: 1735698683 + nsecs: 719378471 +frame_id: "camera_color_optical_frame" +[INFO] [1735698684.224319]: received mask_obs with header seq: 1996 +stamp: + secs: 1735698684 + nsecs: 227669239 +frame_id: '' and grasped_object_mask header seq: 179674 +stamp: + secs: 1735698683 + nsecs: 987030268 +frame_id: "camera_color_optical_frame" +[INFO] [1735698684.484355]: received mask_obs with header seq: 1997 +stamp: + secs: 1735698684 + nsecs: 487869024 +frame_id: '' and grasped_object_mask header seq: 179682 +stamp: + secs: 1735698684 + nsecs: 253481865 +frame_id: "camera_color_optical_frame" +[INFO] [1735698684.746589]: received mask_obs with header seq: 1998 +stamp: + secs: 1735698684 + nsecs: 750021457 +frame_id: '' and grasped_object_mask header seq: 179690 +stamp: + secs: 1735698684 + nsecs: 519933701 +frame_id: "camera_color_optical_frame" +[INFO] [1735698685.010009]: received mask_obs with header seq: 1999 +stamp: + secs: 1735698685 + nsecs: 13477087 +frame_id: '' and grasped_object_mask header seq: 179698 +stamp: + secs: 1735698684 + nsecs: 786386728 +frame_id: "camera_color_optical_frame" +[INFO] [1735698685.265654]: received mask_obs with header seq: 2000 +stamp: + secs: 1735698685 + nsecs: 269018173 +frame_id: '' and grasped_object_mask header seq: 179705 +stamp: + secs: 1735698685 + nsecs: 19530773 +frame_id: "camera_color_optical_frame" +[INFO] [1735698685.522839]: received mask_obs with header seq: 2001 +stamp: + secs: 1735698685 + nsecs: 526463985 +frame_id: '' and grasped_object_mask header seq: 179713 +stamp: + secs: 1735698685 + nsecs: 285980463 +frame_id: "camera_color_optical_frame" +[INFO] [1735698685.781620]: received mask_obs with header seq: 2002 +stamp: + secs: 1735698685 + nsecs: 785205364 +frame_id: '' and grasped_object_mask header seq: 179721 +stamp: + secs: 1735698685 + nsecs: 552431107 +frame_id: "camera_color_optical_frame" +[INFO] [1735698686.045673]: received mask_obs with header seq: 2003 +stamp: + secs: 1735698686 + nsecs: 48322916 +frame_id: '' and grasped_object_mask header seq: 179729 +stamp: + secs: 1735698685 + nsecs: 818880796 +frame_id: "camera_color_optical_frame" +[INFO] [1735698686.307896]: received mask_obs with header seq: 2004 +stamp: + secs: 1735698686 + nsecs: 311345815 +frame_id: '' and grasped_object_mask header seq: 179737 +stamp: + secs: 1735698686 + nsecs: 85329056 +frame_id: "camera_color_optical_frame" +[INFO] [1735698686.563475]: received mask_obs with header seq: 2005 +stamp: + secs: 1735698686 + nsecs: 567123174 +frame_id: '' and grasped_object_mask header seq: 179744 +stamp: + secs: 1735698686 + nsecs: 318471193 +frame_id: "camera_color_optical_frame" +[INFO] [1735698686.835488]: received mask_obs with header seq: 2006 +stamp: + secs: 1735698686 + nsecs: 837558269 +frame_id: '' and grasped_object_mask header seq: 179752 +stamp: + secs: 1735698686 + nsecs: 584918737 +frame_id: "camera_color_optical_frame" +[INFO] [1735698687.108472]: received mask_obs with header seq: 2007 +stamp: + secs: 1735698687 + nsecs: 112063169 +frame_id: '' and grasped_object_mask header seq: 179760 +stamp: + secs: 1735698686 + nsecs: 851366043 +frame_id: "camera_color_optical_frame" +[INFO] [1735698687.357444]: received mask_obs with header seq: 2008 +stamp: + secs: 1735698687 + nsecs: 361047267 +frame_id: '' and grasped_object_mask header seq: 179768 +stamp: + secs: 1735698687 + nsecs: 117812157 +frame_id: "camera_color_optical_frame" +[INFO] [1735698687.616439]: received mask_obs with header seq: 2009 +stamp: + secs: 1735698687 + nsecs: 619851112 +frame_id: '' and grasped_object_mask header seq: 179776 +stamp: + secs: 1735698687 + nsecs: 384259462 +frame_id: "camera_color_optical_frame" +[INFO] [1735698687.877011]: received mask_obs with header seq: 2010 +stamp: + secs: 1735698687 + nsecs: 880589723 +frame_id: '' and grasped_object_mask header seq: 179784 +stamp: + secs: 1735698687 + nsecs: 650705338 +frame_id: "camera_color_optical_frame" +[INFO] [1735698688.142467]: received mask_obs with header seq: 2011 +stamp: + secs: 1735698688 + nsecs: 145421028 +frame_id: '' and grasped_object_mask header seq: 179792 +stamp: + secs: 1735698687 + nsecs: 917150974 +frame_id: "camera_color_optical_frame" +[INFO] [1735698688.394680]: received mask_obs with header seq: 2012 +stamp: + secs: 1735698688 + nsecs: 398201942 +frame_id: '' and grasped_object_mask header seq: 179799 +stamp: + secs: 1735698688 + nsecs: 150292873 +frame_id: "camera_color_optical_frame" +[INFO] [1735698688.653967]: received mask_obs with header seq: 2013 +stamp: + secs: 1735698688 + nsecs: 657593011 +frame_id: '' and grasped_object_mask header seq: 179807 +stamp: + secs: 1735698688 + nsecs: 416739941 +frame_id: "camera_color_optical_frame" +[INFO] [1735698688.914809]: received mask_obs with header seq: 2014 +stamp: + secs: 1735698688 + nsecs: 918403625 +frame_id: '' and grasped_object_mask header seq: 179815 +stamp: + secs: 1735698688 + nsecs: 683186531 +frame_id: "camera_color_optical_frame" +[INFO] [1735698689.177609]: received mask_obs with header seq: 2015 +stamp: + secs: 1735698689 + nsecs: 181090593 +frame_id: '' and grasped_object_mask header seq: 179823 +stamp: + secs: 1735698688 + nsecs: 949631929 +frame_id: "camera_color_optical_frame" +[INFO] [1735698689.444629]: received mask_obs with header seq: 2016 +stamp: + secs: 1735698689 + nsecs: 442175626 +frame_id: '' and grasped_object_mask header seq: 179831 +stamp: + secs: 1735698689 + nsecs: 216073990 +frame_id: "camera_color_optical_frame" +[INFO] [1735698689.711657]: received mask_obs with header seq: 2017 +stamp: + secs: 1735698689 + nsecs: 715302228 +frame_id: '' and grasped_object_mask header seq: 179839 +stamp: + secs: 1735698689 + nsecs: 482514381 +frame_id: "camera_color_optical_frame" +[INFO] [1735698689.973832]: received mask_obs with header seq: 2018 +stamp: + secs: 1735698689 + nsecs: 977190732 +frame_id: '' and grasped_object_mask header seq: 179847 +stamp: + secs: 1735698689 + nsecs: 748954058 +frame_id: "camera_color_optical_frame" +[INFO] [1735698690.242762]: received mask_obs with header seq: 2019 +stamp: + secs: 1735698690 + nsecs: 246333599 +frame_id: '' and grasped_object_mask header seq: 179854 +stamp: + secs: 1735698689 + nsecs: 982091427 +frame_id: "camera_color_optical_frame" +[INFO] [1735698690.510997]: received mask_obs with header seq: 2020 +stamp: + secs: 1735698690 + nsecs: 514486312 +frame_id: '' and grasped_object_mask header seq: 179863 +stamp: + secs: 1735698690 + nsecs: 281842470 +frame_id: "camera_color_optical_frame" +[INFO] [1735698690.773229]: received mask_obs with header seq: 2021 +stamp: + secs: 1735698690 + nsecs: 776847362 +frame_id: '' and grasped_object_mask header seq: 179870 +stamp: + secs: 1735698690 + nsecs: 514982939 +frame_id: "camera_color_optical_frame" +[INFO] [1735698691.025041]: received mask_obs with header seq: 2022 +stamp: + secs: 1735698691 + nsecs: 28708934 +frame_id: '' and grasped_object_mask header seq: 179878 +stamp: + secs: 1735698690 + nsecs: 781427622 +frame_id: "camera_color_optical_frame" +[INFO] [1735698691.284352]: received mask_obs with header seq: 2023 +stamp: + secs: 1735698691 + nsecs: 287806272 +frame_id: '' and grasped_object_mask header seq: 179886 +stamp: + secs: 1735698691 + nsecs: 47879457 +frame_id: "camera_color_optical_frame" +[INFO] [1735698691.555605]: received mask_obs with header seq: 2024 +stamp: + secs: 1735698691 + nsecs: 558991193 +frame_id: '' and grasped_object_mask header seq: 179894 +stamp: + secs: 1735698691 + nsecs: 314334393 +frame_id: "camera_color_optical_frame" +[INFO] [1735698691.823381]: received mask_obs with header seq: 2025 +stamp: + secs: 1735698691 + nsecs: 826572895 +frame_id: '' and grasped_object_mask header seq: 179902 +stamp: + secs: 1735698691 + nsecs: 580789566 +frame_id: "camera_color_optical_frame" +[INFO] [1735698692.084027]: received mask_obs with header seq: 2026 +stamp: + secs: 1735698692 + nsecs: 87584733 +frame_id: '' and grasped_object_mask header seq: 179910 +stamp: + secs: 1735698691 + nsecs: 847245216 +frame_id: "camera_color_optical_frame" +[INFO] [1735698692.344210]: received mask_obs with header seq: 2027 +stamp: + secs: 1735698692 + nsecs: 347854852 +frame_id: '' and grasped_object_mask header seq: 179918 +stamp: + secs: 1735698692 + nsecs: 113689184 +frame_id: "camera_color_optical_frame" +[INFO] [1735698692.611545]: received mask_obs with header seq: 2028 +stamp: + secs: 1735698692 + nsecs: 615085124 +frame_id: '' and grasped_object_mask header seq: 179926 +stamp: + secs: 1735698692 + nsecs: 380131245 +frame_id: "camera_color_optical_frame" +[INFO] [1735698692.871142]: received mask_obs with header seq: 2029 +stamp: + secs: 1735698692 + nsecs: 874642133 +frame_id: '' and grasped_object_mask header seq: 179934 +stamp: + secs: 1735698692 + nsecs: 646572113 +frame_id: "camera_color_optical_frame" +[INFO] [1735698693.128990]: received mask_obs with header seq: 2030 +stamp: + secs: 1735698693 + nsecs: 132579803 +frame_id: '' and grasped_object_mask header seq: 179941 +stamp: + secs: 1735698692 + nsecs: 879707336 +frame_id: "camera_color_optical_frame" +[INFO] [1735698693.384787]: received mask_obs with header seq: 2031 +stamp: + secs: 1735698693 + nsecs: 388195753 +frame_id: '' and grasped_object_mask header seq: 179949 +stamp: + secs: 1735698693 + nsecs: 146167755 +frame_id: "camera_color_optical_frame" +[INFO] [1735698693.645486]: received mask_obs with header seq: 2032 +stamp: + secs: 1735698693 + nsecs: 649071455 +frame_id: '' and grasped_object_mask header seq: 179957 +stamp: + secs: 1735698693 + nsecs: 412629128 +frame_id: "camera_color_optical_frame" +[INFO] [1735698693.908540]: received mask_obs with header seq: 2033 +stamp: + secs: 1735698693 + nsecs: 912201166 +frame_id: '' and grasped_object_mask header seq: 179965 +stamp: + secs: 1735698693 + nsecs: 679090977 +frame_id: "camera_color_optical_frame" +[INFO] [1735698694.169172]: received mask_obs with header seq: 2034 +stamp: + secs: 1735698694 + nsecs: 172229528 +frame_id: '' and grasped_object_mask header seq: 179973 +stamp: + secs: 1735698693 + nsecs: 945550919 +frame_id: "camera_color_optical_frame" +[INFO] [1735698694.421782]: received mask_obs with header seq: 2035 +stamp: + secs: 1735698694 + nsecs: 425249099 +frame_id: '' and grasped_object_mask header seq: 179980 +stamp: + secs: 1735698694 + nsecs: 178696156 +frame_id: "camera_color_optical_frame" +[INFO] [1735698694.682445]: received mask_obs with header seq: 2036 +stamp: + secs: 1735698694 + nsecs: 686071157 +frame_id: '' and grasped_object_mask header seq: 179988 +stamp: + secs: 1735698694 + nsecs: 445147038 +frame_id: "camera_color_optical_frame" +[INFO] [1735698694.945560]: received mask_obs with header seq: 2037 +stamp: + secs: 1735698694 + nsecs: 948946714 +frame_id: '' and grasped_object_mask header seq: 179996 +stamp: + secs: 1735698694 + nsecs: 711598396 +frame_id: "camera_color_optical_frame" +[INFO] [1735698695.207953]: received mask_obs with header seq: 2038 +stamp: + secs: 1735698695 + nsecs: 211429357 +frame_id: '' and grasped_object_mask header seq: 180004 +stamp: + secs: 1735698694 + nsecs: 978326082 +frame_id: "camera_color_optical_frame" +[INFO] [1735698695.471172]: received mask_obs with header seq: 2039 +stamp: + secs: 1735698695 + nsecs: 474797487 +frame_id: '' and grasped_object_mask header seq: 180012 +stamp: + secs: 1735698695 + nsecs: 247871637 +frame_id: "camera_color_optical_frame" +[INFO] [1735698695.723789]: received mask_obs with header seq: 2040 +stamp: + secs: 1735698695 + nsecs: 727380514 +frame_id: '' and grasped_object_mask header seq: 180019 +stamp: + secs: 1735698695 + nsecs: 483850718 +frame_id: "camera_color_optical_frame" +[INFO] [1735698695.983669]: received mask_obs with header seq: 2041 +stamp: + secs: 1735698695 + nsecs: 987240552 +frame_id: '' and grasped_object_mask header seq: 180027 +stamp: + secs: 1735698695 + nsecs: 753685713 +frame_id: "camera_color_optical_frame" +[INFO] [1735698696.244412]: received mask_obs with header seq: 2042 +stamp: + secs: 1735698696 + nsecs: 248096704 +frame_id: '' and grasped_object_mask header seq: 180035 +stamp: + secs: 1735698696 + nsecs: 22839785 +frame_id: "camera_color_optical_frame" +[INFO] [1735698696.508606]: received mask_obs with header seq: 2043 +stamp: + secs: 1735698696 + nsecs: 512189865 +frame_id: '' and grasped_object_mask header seq: 180043 +stamp: + secs: 1735698696 + nsecs: 288992643 +frame_id: "camera_color_optical_frame" +[INFO] [1735698696.778043]: received mask_obs with header seq: 2044 +stamp: + secs: 1735698696 + nsecs: 781212329 +frame_id: '' and grasped_object_mask header seq: 180051 +stamp: + secs: 1735698696 + nsecs: 555125952 +frame_id: "camera_color_optical_frame" +[INFO] [1735698697.040971]: received mask_obs with header seq: 2045 +stamp: + secs: 1735698697 + nsecs: 44479846 +frame_id: '' and grasped_object_mask header seq: 180059 +stamp: + secs: 1735698696 + nsecs: 821238279 +frame_id: "camera_color_optical_frame" +[INFO] [1735698697.297671]: received mask_obs with header seq: 2046 +stamp: + secs: 1735698697 + nsecs: 300146579 +frame_id: '' and grasped_object_mask header seq: 180066 +stamp: + secs: 1735698697 + nsecs: 54089069 +frame_id: "camera_color_optical_frame" +[INFO] [1735698697.556041]: received mask_obs with header seq: 2047 +stamp: + secs: 1735698697 + nsecs: 559720754 +frame_id: '' and grasped_object_mask header seq: 180074 +stamp: + secs: 1735698697 + nsecs: 320222378 +frame_id: "camera_color_optical_frame" +[INFO] [1735698697.812068]: received mask_obs with header seq: 2048 +stamp: + secs: 1735698697 + nsecs: 815766811 +frame_id: '' and grasped_object_mask header seq: 180082 +stamp: + secs: 1735698697 + nsecs: 586333513 +frame_id: "camera_color_optical_frame" +[INFO] [1735698698.082002]: received mask_obs with header seq: 2049 +stamp: + secs: 1735698698 + nsecs: 85517883 +frame_id: '' and grasped_object_mask header seq: 180090 +stamp: + secs: 1735698697 + nsecs: 852422953 +frame_id: "camera_color_optical_frame" +[INFO] [1735698698.345159]: received mask_obs with header seq: 2050 +stamp: + secs: 1735698698 + nsecs: 348768711 +frame_id: '' and grasped_object_mask header seq: 180098 +stamp: + secs: 1735698698 + nsecs: 118539095 +frame_id: "camera_color_optical_frame" +[INFO] [1735698698.609427]: received mask_obs with header seq: 2051 +stamp: + secs: 1735698698 + nsecs: 612861871 +frame_id: '' and grasped_object_mask header seq: 180106 +stamp: + secs: 1735698698 + nsecs: 384668827 +frame_id: "camera_color_optical_frame" +[INFO] [1735698698.872789]: received mask_obs with header seq: 2052 +stamp: + secs: 1735698698 + nsecs: 876458168 +frame_id: '' and grasped_object_mask header seq: 180114 +stamp: + secs: 1735698698 + nsecs: 650776863 +frame_id: "camera_color_optical_frame" +[INFO] [1735698699.132837]: received mask_obs with header seq: 2053 +stamp: + secs: 1735698699 + nsecs: 136528968 +frame_id: '' and grasped_object_mask header seq: 180121 +stamp: + secs: 1735698698 + nsecs: 883603811 +frame_id: "camera_color_optical_frame" +[INFO] [1735698699.391210]: received mask_obs with header seq: 2054 +stamp: + secs: 1735698699 + nsecs: 394875764 +frame_id: '' and grasped_object_mask header seq: 180129 +stamp: + secs: 1735698699 + nsecs: 149745464 +frame_id: "camera_color_optical_frame" +[INFO] [1735698699.650766]: received mask_obs with header seq: 2055 +stamp: + secs: 1735698699 + nsecs: 654441356 +frame_id: '' and grasped_object_mask header seq: 180137 +stamp: + secs: 1735698699 + nsecs: 415895939 +frame_id: "camera_color_optical_frame" +[INFO] [1735698699.911150]: received mask_obs with header seq: 2056 +stamp: + secs: 1735698699 + nsecs: 914706468 +frame_id: '' and grasped_object_mask header seq: 180145 +stamp: + secs: 1735698699 + nsecs: 682027817 +frame_id: "camera_color_optical_frame" +[INFO] [1735698700.173765]: received mask_obs with header seq: 2057 +stamp: + secs: 1735698700 + nsecs: 177363634 +frame_id: '' and grasped_object_mask header seq: 180153 +stamp: + secs: 1735698699 + nsecs: 948140383 +frame_id: "camera_color_optical_frame" +[INFO] [1735698700.437422]: received mask_obs with header seq: 2058 +stamp: + secs: 1735698700 + nsecs: 441059350 +frame_id: '' and grasped_object_mask header seq: 180161 +stamp: + secs: 1735698700 + nsecs: 214303970 +frame_id: "camera_color_optical_frame" +[INFO] [1735698700.690166]: received mask_obs with header seq: 2059 +stamp: + secs: 1735698700 + nsecs: 693727731 +frame_id: '' and grasped_object_mask header seq: 180168 +stamp: + secs: 1735698700 + nsecs: 447181940 +frame_id: "camera_color_optical_frame" +[INFO] [1735698700.948885]: received mask_obs with header seq: 2060 +stamp: + secs: 1735698700 + nsecs: 952508687 +frame_id: '' and grasped_object_mask header seq: 180176 +stamp: + secs: 1735698700 + nsecs: 713309765 +frame_id: "camera_color_optical_frame" +[INFO] [1735698701.208118]: received mask_obs with header seq: 2061 +stamp: + secs: 1735698701 + nsecs: 211343288 +frame_id: '' and grasped_object_mask header seq: 180184 +stamp: + secs: 1735698700 + nsecs: 979422808 +frame_id: "camera_color_optical_frame" +[INFO] [1735698701.471035]: received mask_obs with header seq: 2062 +stamp: + secs: 1735698701 + nsecs: 474651336 +frame_id: '' and grasped_object_mask header seq: 180192 +stamp: + secs: 1735698701 + nsecs: 245563269 +frame_id: "camera_color_optical_frame" +[INFO] [1735698701.733704]: received mask_obs with header seq: 2063 +stamp: + secs: 1735698701 + nsecs: 737328290 +frame_id: '' and grasped_object_mask header seq: 180199 +stamp: + secs: 1735698701 + nsecs: 478419542 +frame_id: "camera_color_optical_frame" +[INFO] [1735698702.006599]: received mask_obs with header seq: 2064 +stamp: + secs: 1735698702 + nsecs: 9718894 +frame_id: '' and grasped_object_mask header seq: 180207 +stamp: + secs: 1735698701 + nsecs: 744520187 +frame_id: "camera_color_optical_frame" +[INFO] [1735698702.269260]: received mask_obs with header seq: 2065 +stamp: + secs: 1735698702 + nsecs: 272912740 +frame_id: '' and grasped_object_mask header seq: 180216 +stamp: + secs: 1735698702 + nsecs: 43885469 +frame_id: "camera_color_optical_frame" +[INFO] [1735698702.521038]: received mask_obs with header seq: 2066 +stamp: + secs: 1735698702 + nsecs: 524730920 +frame_id: '' and grasped_object_mask header seq: 180223 +stamp: + secs: 1735698702 + nsecs: 276759386 +frame_id: "camera_color_optical_frame" +[INFO] [1735698702.783076]: received mask_obs with header seq: 2067 +stamp: + secs: 1735698702 + nsecs: 786086559 +frame_id: '' and grasped_object_mask header seq: 180231 +stamp: + secs: 1735698702 + nsecs: 542880297 +frame_id: "camera_color_optical_frame" +[INFO] [1735698703.041538]: received mask_obs with header seq: 2068 +stamp: + secs: 1735698703 + nsecs: 44975042 +frame_id: '' and grasped_object_mask header seq: 180239 +stamp: + secs: 1735698702 + nsecs: 808981657 +frame_id: "camera_color_optical_frame" +[INFO] [1735698703.310463]: received mask_obs with header seq: 2069 +stamp: + secs: 1735698703 + nsecs: 314073085 +frame_id: '' and grasped_object_mask header seq: 180247 +stamp: + secs: 1735698703 + nsecs: 75097799 +frame_id: "camera_color_optical_frame" +[INFO] [1735698703.572656]: received mask_obs with header seq: 2070 +stamp: + secs: 1735698703 + nsecs: 576308012 +frame_id: '' and grasped_object_mask header seq: 180255 +stamp: + secs: 1735698703 + nsecs: 341238260 +frame_id: "camera_color_optical_frame" +[INFO] [1735698703.836372]: received mask_obs with header seq: 2071 +stamp: + secs: 1735698703 + nsecs: 839856863 +frame_id: '' and grasped_object_mask header seq: 180263 +stamp: + secs: 1735698703 + nsecs: 607357979 +frame_id: "camera_color_optical_frame" +[INFO] [1735698704.088741]: received mask_obs with header seq: 2072 +stamp: + secs: 1735698704 + nsecs: 92135429 +frame_id: '' and grasped_object_mask header seq: 180270 +stamp: + secs: 1735698703 + nsecs: 840196133 +frame_id: "camera_color_optical_frame" +[INFO] [1735698704.349639]: received mask_obs with header seq: 2073 +stamp: + secs: 1735698704 + nsecs: 352850198 +frame_id: '' and grasped_object_mask header seq: 180278 +stamp: + secs: 1735698704 + nsecs: 106321335 +frame_id: "camera_color_optical_frame" +[INFO] [1735698704.607138]: received mask_obs with header seq: 2074 +stamp: + secs: 1735698704 + nsecs: 610395669 +frame_id: '' and grasped_object_mask header seq: 180286 +stamp: + secs: 1735698704 + nsecs: 372458696 +frame_id: "camera_color_optical_frame" +[INFO] [1735698704.849667]: received mask_obs with header seq: 2075 +stamp: + secs: 1735698704 + nsecs: 853310108 +frame_id: '' and grasped_object_mask header seq: 180293 +stamp: + secs: 1735698704 + nsecs: 605312586 +frame_id: "camera_color_optical_frame" +[INFO] [1735698705.109873]: received mask_obs with header seq: 2076 +stamp: + secs: 1735698705 + nsecs: 113512992 +frame_id: '' and grasped_object_mask header seq: 180301 +stamp: + secs: 1735698704 + nsecs: 871411085 +frame_id: "camera_color_optical_frame" +[INFO] [1735698705.370994]: received mask_obs with header seq: 2077 +stamp: + secs: 1735698705 + nsecs: 374568700 +frame_id: '' and grasped_object_mask header seq: 180309 +stamp: + secs: 1735698705 + nsecs: 137542486 +frame_id: "camera_color_optical_frame" +[INFO] [1735698705.628904]: received mask_obs with header seq: 2078 +stamp: + secs: 1735698705 + nsecs: 631697177 +frame_id: '' and grasped_object_mask header seq: 180316 +stamp: + secs: 1735698705 + nsecs: 370408773 +frame_id: "camera_color_optical_frame" +[INFO] [1735698705.883796]: received mask_obs with header seq: 2079 +stamp: + secs: 1735698705 + nsecs: 887416362 +frame_id: '' and grasped_object_mask header seq: 180324 +stamp: + secs: 1735698705 + nsecs: 636522293 +frame_id: "camera_color_optical_frame" +[INFO] [1735698706.140671]: received mask_obs with header seq: 2080 +stamp: + secs: 1735698706 + nsecs: 144319057 +frame_id: '' and grasped_object_mask header seq: 180332 +stamp: + secs: 1735698705 + nsecs: 902616024 +frame_id: "camera_color_optical_frame" +[INFO] [1735698706.405332]: received mask_obs with header seq: 2081 +stamp: + secs: 1735698706 + nsecs: 408862829 +frame_id: '' and grasped_object_mask header seq: 180340 +stamp: + secs: 1735698706 + nsecs: 168751001 +frame_id: "camera_color_optical_frame" +[INFO] [1735698706.657346]: received mask_obs with header seq: 2082 +stamp: + secs: 1735698706 + nsecs: 660894870 +frame_id: '' and grasped_object_mask header seq: 180347 +stamp: + secs: 1735698706 + nsecs: 401614428 +frame_id: "camera_color_optical_frame" +[INFO] [1735698706.917108]: received mask_obs with header seq: 2083 +stamp: + secs: 1735698706 + nsecs: 920759439 +frame_id: '' and grasped_object_mask header seq: 180355 +stamp: + secs: 1735698706 + nsecs: 667723894 +frame_id: "camera_color_optical_frame" +[INFO] [1735698707.177392]: received mask_obs with header seq: 2084 +stamp: + secs: 1735698707 + nsecs: 180593013 +frame_id: '' and grasped_object_mask header seq: 180363 +stamp: + secs: 1735698706 + nsecs: 933812618 +frame_id: "camera_color_optical_frame" +[INFO] [1735698707.435949]: received mask_obs with header seq: 2085 +stamp: + secs: 1735698707 + nsecs: 439317226 +frame_id: '' and grasped_object_mask header seq: 180371 +stamp: + secs: 1735698707 + nsecs: 199963808 +frame_id: "camera_color_optical_frame" +[INFO] [1735698707.689637]: received mask_obs with header seq: 2086 +stamp: + secs: 1735698707 + nsecs: 693300962 +frame_id: '' and grasped_object_mask header seq: 180378 +stamp: + secs: 1735698707 + nsecs: 432834625 +frame_id: "camera_color_optical_frame" +[INFO] [1735698707.948669]: received mask_obs with header seq: 2087 +stamp: + secs: 1735698707 + nsecs: 952272415 +frame_id: '' and grasped_object_mask header seq: 180386 +stamp: + secs: 1735698707 + nsecs: 698952913 +frame_id: "camera_color_optical_frame" +[INFO] [1735698708.214682]: received mask_obs with header seq: 2088 +stamp: + secs: 1735698708 + nsecs: 218307733 +frame_id: '' and grasped_object_mask header seq: 180394 +stamp: + secs: 1735698707 + nsecs: 965054989 +frame_id: "camera_color_optical_frame" +[INFO] [1735698708.474673]: received mask_obs with header seq: 2089 +stamp: + secs: 1735698708 + nsecs: 478321552 +frame_id: '' and grasped_object_mask header seq: 180402 +stamp: + secs: 1735698708 + nsecs: 231198788 +frame_id: "camera_color_optical_frame" +[INFO] [1735698708.736646]: received mask_obs with header seq: 2090 +stamp: + secs: 1735698708 + nsecs: 740262985 +frame_id: '' and grasped_object_mask header seq: 180410 +stamp: + secs: 1735698708 + nsecs: 497320175 +frame_id: "camera_color_optical_frame" +[INFO] [1735698709.001406]: received mask_obs with header seq: 2091 +stamp: + secs: 1735698709 + nsecs: 4902601 +frame_id: '' and grasped_object_mask header seq: 180418 +stamp: + secs: 1735698708 + nsecs: 763420105 +frame_id: "camera_color_optical_frame" +[INFO] [1735698709.250690]: received mask_obs with header seq: 2092 +stamp: + secs: 1735698709 + nsecs: 254346609 +frame_id: '' and grasped_object_mask header seq: 180425 +stamp: + secs: 1735698708 + nsecs: 996257782 +frame_id: "camera_color_optical_frame" +[INFO] [1735698709.359919]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit[pub_text-10] killing on exit +[INFO] [1735698709.515991]: received mask_obs with header seq: 2093 +stamp: + secs: 1735698709 + nsecs: 519675970 +frame_id: '' and grasped_object_mask header seq: 180433 +stamp: + secs: 1735698709 + nsecs: 262408495 +frame_id: "camera_color_optical_frame" +[INFO] [1735698710.141567]: Stored episode 4. +[INFO] [1735698710.153857]: Saving video... +[INFO] [1735698710.373593]: Video saved! +Starting episode 4 +[INFO] [1735698710.440028]: resetting environment +[INFO] [1735698710.460347]: cleared current plan +[INFO] [1735698710.460548]: moving to home +[INFO] [1735698714.064864]: reached home +[INFO] [1735698714.073378]: reset action history +[INFO] [1735698716.870977]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735698823.835924]: resetting environment +[INFO] [1735698823.873063]: cleared current plan +[INFO] [1735698823.877314]: moving to home +[INFO] [1735698824.882662]: reached home +[INFO] [1735698824.888928]: reset action history +[INFO] [1735698827.684406]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/episode_rosbags/episode_4_2024-12-31-21-33-47.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1746382.log +started roslaunch server http://158.130.50.37:46017/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1747837] +process[print_text-14]: started with pid [1747838] +process[rosbag_record-15]: started with pid [1747839] +started bagging! +For topic gripper_width: timestamp difference is 118153640 for nearest: 1735698830453393289 - target: 1735698830335239649 at idx 34 +[INFO] [1735698830.567627]: received mask_obs with header seq: 2094 +stamp: + secs: 1735698830 + nsecs: 571015357 +frame_id: '' and grasped_object_mask header seq: 184068 +stamp: + secs: 1735698830 + nsecs: 335239649 +frame_id: "camera_color_optical_frame" +[INFO] [1735698830.809609]: received mask_obs with header seq: 2095 +stamp: + secs: 1735698830 + nsecs: 813043355 +frame_id: '' and grasped_object_mask header seq: 184075 +stamp: + secs: 1735698830 + nsecs: 568386078 +frame_id: "camera_color_optical_frame" +[INFO] [1735698831.045982]: received mask_obs with header seq: 2096 +stamp: + secs: 1735698831 + nsecs: 49535989 +frame_id: '' and grasped_object_mask header seq: 184083 +stamp: + secs: 1735698830 + nsecs: 834838390 +frame_id: "camera_color_optical_frame" +[INFO] [1735698831.284895]: received mask_obs with header seq: 2097 +stamp: + secs: 1735698831 + nsecs: 288507461 +frame_id: '' and grasped_object_mask header seq: 184090 +stamp: + secs: 1735698831 + nsecs: 67984581 +frame_id: "camera_color_optical_frame" +[INFO] [1735698831.529698]: received mask_obs with header seq: 2098 +stamp: + secs: 1735698831 + nsecs: 533336400 +frame_id: '' and grasped_object_mask header seq: 184097 +stamp: + secs: 1735698831 + nsecs: 301131487 +frame_id: "camera_color_optical_frame" +[INFO] [1735698831.777593]: received mask_obs with header seq: 2099 +stamp: + secs: 1735698831 + nsecs: 781239032 +frame_id: '' and grasped_object_mask header seq: 184104 +stamp: + secs: 1735698831 + nsecs: 534278393 +frame_id: "camera_color_optical_frame" +[INFO] [1735698832.017317]: received mask_obs with header seq: 2100 +stamp: + secs: 1735698832 + nsecs: 20874261 +frame_id: '' and grasped_object_mask header seq: 184111 +stamp: + secs: 1735698831 + nsecs: 767426014 +frame_id: "camera_color_optical_frame" +[INFO] [1735698832.282533]: received mask_obs with header seq: 2101 +stamp: + secs: 1735698832 + nsecs: 285939931 +frame_id: '' and grasped_object_mask header seq: 184119 +stamp: + secs: 1735698832 + nsecs: 33880234 +frame_id: "camera_color_optical_frame" +[INFO] [1735698832.542234]: received mask_obs with header seq: 2102 +stamp: + secs: 1735698832 + nsecs: 545568227 +frame_id: '' and grasped_object_mask header seq: 184127 +stamp: + secs: 1735698832 + nsecs: 300337553 +frame_id: "camera_color_optical_frame" +[INFO] [1735698832.802877]: received mask_obs with header seq: 2103 +stamp: + secs: 1735698832 + nsecs: 805586338 +frame_id: '' and grasped_object_mask header seq: 184135 +stamp: + secs: 1735698832 + nsecs: 566798925 +frame_id: "camera_color_optical_frame" +[INFO] [1735698833.068130]: received mask_obs with header seq: 2104 +stamp: + secs: 1735698833 + nsecs: 71765184 +frame_id: '' and grasped_object_mask header seq: 184143 +stamp: + secs: 1735698832 + nsecs: 833260536 +frame_id: "camera_color_optical_frame" +[INFO] [1735698833.331317]: received mask_obs with header seq: 2105 +stamp: + secs: 1735698833 + nsecs: 334561586 +frame_id: '' and grasped_object_mask header seq: 184150 +stamp: + secs: 1735698833 + nsecs: 66415071 +frame_id: "camera_color_optical_frame" +[INFO] [1735698833.586840]: received mask_obs with header seq: 2106 +stamp: + secs: 1735698833 + nsecs: 590466976 +frame_id: '' and grasped_object_mask header seq: 184158 +stamp: + secs: 1735698833 + nsecs: 332879543 +frame_id: "camera_color_optical_frame" +[INFO] [1735698833.846244]: received mask_obs with header seq: 2107 +stamp: + secs: 1735698833 + nsecs: 849869966 +frame_id: '' and grasped_object_mask header seq: 184166 +stamp: + secs: 1735698833 + nsecs: 599348783 +frame_id: "camera_color_optical_frame" +[INFO] [1735698834.108149]: received mask_obs with header seq: 2108 +stamp: + secs: 1735698834 + nsecs: 111791372 +frame_id: '' and grasped_object_mask header seq: 184174 +stamp: + secs: 1735698833 + nsecs: 865818739 +frame_id: "camera_color_optical_frame" +[INFO] [1735698834.367437]: received mask_obs with header seq: 2109 +stamp: + secs: 1735698834 + nsecs: 371088981 +frame_id: '' and grasped_object_mask header seq: 184182 +stamp: + secs: 1735698834 + nsecs: 132290840 +frame_id: "camera_color_optical_frame" +[INFO] [1735698834.640398]: received mask_obs with header seq: 2110 +stamp: + secs: 1735698834 + nsecs: 644050598 +frame_id: '' and grasped_object_mask header seq: 184190 +stamp: + secs: 1735698834 + nsecs: 398754358 +frame_id: "camera_color_optical_frame" +[INFO] [1735698834.901681]: received mask_obs with header seq: 2111 +stamp: + secs: 1735698834 + nsecs: 905185937 +frame_id: '' and grasped_object_mask header seq: 184198 +stamp: + secs: 1735698834 + nsecs: 665215015 +frame_id: "camera_color_optical_frame" +[INFO] [1735698835.167697]: received mask_obs with header seq: 2112 +stamp: + secs: 1735698835 + nsecs: 170942068 +frame_id: '' and grasped_object_mask header seq: 184206 +stamp: + secs: 1735698834 + nsecs: 931676388 +frame_id: "camera_color_optical_frame" +[INFO] [1735698835.423476]: received mask_obs with header seq: 2113 +stamp: + secs: 1735698835 + nsecs: 427116394 +frame_id: '' and grasped_object_mask header seq: 184213 +stamp: + secs: 1735698835 + nsecs: 164830685 +frame_id: "camera_color_optical_frame" +[INFO] [1735698835.685784]: received mask_obs with header seq: 2114 +stamp: + secs: 1735698835 + nsecs: 689357519 +frame_id: '' and grasped_object_mask header seq: 184222 +stamp: + secs: 1735698835 + nsecs: 464593887 +frame_id: "camera_color_optical_frame" +[INFO] [1735698835.937063]: received mask_obs with header seq: 2115 +stamp: + secs: 1735698835 + nsecs: 940689086 +frame_id: '' and grasped_object_mask header seq: 184229 +stamp: + secs: 1735698835 + nsecs: 697741508 +frame_id: "camera_color_optical_frame" +[INFO] [1735698836.194637]: received mask_obs with header seq: 2116 +stamp: + secs: 1735698836 + nsecs: 197247982 +frame_id: '' and grasped_object_mask header seq: 184237 +stamp: + secs: 1735698835 + nsecs: 964196920 +frame_id: "camera_color_optical_frame" +[INFO] [1735698836.457980]: received mask_obs with header seq: 2117 +stamp: + secs: 1735698836 + nsecs: 460752487 +frame_id: '' and grasped_object_mask header seq: 184245 +stamp: + secs: 1735698836 + nsecs: 230651617 +frame_id: "camera_color_optical_frame" +[INFO] [1735698836.723037]: received mask_obs with header seq: 2118 +stamp: + secs: 1735698836 + nsecs: 726566314 +frame_id: '' and grasped_object_mask header seq: 184253 +stamp: + secs: 1735698836 + nsecs: 497103691 +frame_id: "camera_color_optical_frame" +[INFO] [1735698836.990448]: received mask_obs with header seq: 2119 +stamp: + secs: 1735698836 + nsecs: 993732452 +frame_id: '' and grasped_object_mask header seq: 184260 +stamp: + secs: 1735698836 + nsecs: 730249882 +frame_id: "camera_color_optical_frame" +[INFO] [1735698837.253616]: received mask_obs with header seq: 2120 +stamp: + secs: 1735698837 + nsecs: 257273435 +frame_id: '' and grasped_object_mask header seq: 184269 +stamp: + secs: 1735698837 + nsecs: 30008316 +frame_id: "camera_color_optical_frame" +[INFO] [1735698837.504803]: received mask_obs with header seq: 2121 +stamp: + secs: 1735698837 + nsecs: 508397102 +frame_id: '' and grasped_object_mask header seq: 184276 +stamp: + secs: 1735698837 + nsecs: 263154984 +frame_id: "camera_color_optical_frame" +[INFO] [1735698837.767498]: received mask_obs with header seq: 2122 +stamp: + secs: 1735698837 + nsecs: 770933866 +frame_id: '' and grasped_object_mask header seq: 184284 +stamp: + secs: 1735698837 + nsecs: 529610634 +frame_id: "camera_color_optical_frame" +[INFO] [1735698838.025419]: received mask_obs with header seq: 2123 +stamp: + secs: 1735698838 + nsecs: 29054403 +frame_id: '' and grasped_object_mask header seq: 184292 +stamp: + secs: 1735698837 + nsecs: 796067476 +frame_id: "camera_color_optical_frame" +[INFO] [1735698838.287373]: received mask_obs with header seq: 2124 +stamp: + secs: 1735698838 + nsecs: 290943384 +frame_id: '' and grasped_object_mask header seq: 184300 +stamp: + secs: 1735698838 + nsecs: 62523127 +frame_id: "camera_color_optical_frame" +[INFO] [1735698838.551182]: received mask_obs with header seq: 2125 +stamp: + secs: 1735698838 + nsecs: 554795503 +frame_id: '' and grasped_object_mask header seq: 184308 +stamp: + secs: 1735698838 + nsecs: 328975916 +frame_id: "camera_color_optical_frame" +[INFO] [1735698838.802417]: received mask_obs with header seq: 2126 +stamp: + secs: 1735698838 + nsecs: 805863618 +frame_id: '' and grasped_object_mask header seq: 184315 +stamp: + secs: 1735698838 + nsecs: 562118053 +frame_id: "camera_color_optical_frame" +[INFO] [1735698839.060886]: received mask_obs with header seq: 2127 +stamp: + secs: 1735698839 + nsecs: 64337730 +frame_id: '' and grasped_object_mask header seq: 184323 +stamp: + secs: 1735698838 + nsecs: 828565836 +frame_id: "camera_color_optical_frame" +[INFO] [1735698839.321132]: received mask_obs with header seq: 2128 +stamp: + secs: 1735698839 + nsecs: 324143171 +frame_id: '' and grasped_object_mask header seq: 184331 +stamp: + secs: 1735698839 + nsecs: 95013380 +frame_id: "camera_color_optical_frame" +[INFO] [1735698839.589264]: received mask_obs with header seq: 2129 +stamp: + secs: 1735698839 + nsecs: 592562675 +frame_id: '' and grasped_object_mask header seq: 184339 +stamp: + secs: 1735698839 + nsecs: 361459970 +frame_id: "camera_color_optical_frame" +[INFO] [1735698839.840897]: received mask_obs with header seq: 2130 +stamp: + secs: 1735698839 + nsecs: 844443559 +frame_id: '' and grasped_object_mask header seq: 184346 +stamp: + secs: 1735698839 + nsecs: 594600201 +frame_id: "camera_color_optical_frame" +[INFO] [1735698840.099370]: received mask_obs with header seq: 2131 +stamp: + secs: 1735698840 + nsecs: 102682590 +frame_id: '' and grasped_object_mask header seq: 184354 +stamp: + secs: 1735698839 + nsecs: 861045122 +frame_id: "camera_color_optical_frame" +[INFO] [1735698840.355461]: received mask_obs with header seq: 2132 +stamp: + secs: 1735698840 + nsecs: 359027862 +frame_id: '' and grasped_object_mask header seq: 184362 +stamp: + secs: 1735698840 + nsecs: 127490044 +frame_id: "camera_color_optical_frame" +[INFO] [1735698840.620522]: received mask_obs with header seq: 2133 +stamp: + secs: 1735698840 + nsecs: 623074769 +frame_id: '' and grasped_object_mask header seq: 184370 +stamp: + secs: 1735698840 + nsecs: 393939257 +frame_id: "camera_color_optical_frame" +[INFO] [1735698840.888285]: received mask_obs with header seq: 2134 +stamp: + secs: 1735698840 + nsecs: 891644477 +frame_id: '' and grasped_object_mask header seq: 184378 +stamp: + secs: 1735698840 + nsecs: 660389185 +frame_id: "camera_color_optical_frame" +[INFO] [1735698841.152255]: received mask_obs with header seq: 2135 +stamp: + secs: 1735698841 + nsecs: 155814409 +frame_id: '' and grasped_object_mask header seq: 184386 +stamp: + secs: 1735698840 + nsecs: 926838875 +frame_id: "camera_color_optical_frame" +[INFO] [1735698841.416077]: received mask_obs with header seq: 2136 +stamp: + secs: 1735698841 + nsecs: 419686794 +frame_id: '' and grasped_object_mask header seq: 184394 +stamp: + secs: 1735698841 + nsecs: 193288565 +frame_id: "camera_color_optical_frame" +[INFO] [1735698841.667795]: received mask_obs with header seq: 2137 +stamp: + secs: 1735698841 + nsecs: 671126365 +frame_id: '' and grasped_object_mask header seq: 184401 +stamp: + secs: 1735698841 + nsecs: 426429987 +frame_id: "camera_color_optical_frame" +[INFO] [1735698841.930675]: received mask_obs with header seq: 2138 +stamp: + secs: 1735698841 + nsecs: 934333324 +frame_id: '' and grasped_object_mask header seq: 184409 +stamp: + secs: 1735698841 + nsecs: 692875624 +frame_id: "camera_color_optical_frame" +[INFO] [1735698842.196206]: received mask_obs with header seq: 2139 +stamp: + secs: 1735698842 + nsecs: 199782371 +frame_id: '' and grasped_object_mask header seq: 184417 +stamp: + secs: 1735698841 + nsecs: 959321022 +frame_id: "camera_color_optical_frame" +[INFO] [1735698842.454364]: received mask_obs with header seq: 2140 +stamp: + secs: 1735698842 + nsecs: 457655191 +frame_id: '' and grasped_object_mask header seq: 184425 +stamp: + secs: 1735698842 + nsecs: 225767612 +frame_id: "camera_color_optical_frame" +[INFO] [1735698842.717787]: received mask_obs with header seq: 2141 +stamp: + secs: 1735698842 + nsecs: 721339702 +frame_id: '' and grasped_object_mask header seq: 184433 +stamp: + secs: 1735698842 + nsecs: 492221117 +frame_id: "camera_color_optical_frame" +[INFO] [1735698842.981990]: received mask_obs with header seq: 2142 +stamp: + secs: 1735698842 + nsecs: 985580682 +frame_id: '' and grasped_object_mask header seq: 184441 +stamp: + secs: 1735698842 + nsecs: 758674860 +frame_id: "camera_color_optical_frame" +[INFO] [1735698843.236621]: received mask_obs with header seq: 2143 +stamp: + secs: 1735698843 + nsecs: 240175724 +frame_id: '' and grasped_object_mask header seq: 184448 +stamp: + secs: 1735698842 + nsecs: 991822004 +frame_id: "camera_color_optical_frame" +[INFO] [1735698843.496142]: received mask_obs with header seq: 2144 +stamp: + secs: 1735698843 + nsecs: 499739646 +frame_id: '' and grasped_object_mask header seq: 184456 +stamp: + secs: 1735698843 + nsecs: 258276224 +frame_id: "camera_color_optical_frame" +[INFO] [1735698843.756908]: received mask_obs with header seq: 2145 +stamp: + secs: 1735698843 + nsecs: 760488271 +frame_id: '' and grasped_object_mask header seq: 184464 +stamp: + secs: 1735698843 + nsecs: 524732351 +frame_id: "camera_color_optical_frame" +[INFO] [1735698844.020521]: received mask_obs with header seq: 2146 +stamp: + secs: 1735698844 + nsecs: 24111986 +frame_id: '' and grasped_object_mask header seq: 184472 +stamp: + secs: 1735698843 + nsecs: 791188717 +frame_id: "camera_color_optical_frame" +[INFO] [1735698844.285059]: received mask_obs with header seq: 2147 +stamp: + secs: 1735698844 + nsecs: 288529396 +frame_id: '' and grasped_object_mask header seq: 184480 +stamp: + secs: 1735698844 + nsecs: 57646275 +frame_id: "camera_color_optical_frame" +[INFO] [1735698844.548538]: received mask_obs with header seq: 2148 +stamp: + secs: 1735698844 + nsecs: 552181959 +frame_id: '' and grasped_object_mask header seq: 184488 +stamp: + secs: 1735698844 + nsecs: 324100971 +frame_id: "camera_color_optical_frame" +[INFO] [1735698844.819347]: received mask_obs with header seq: 2149 +stamp: + secs: 1735698844 + nsecs: 822752237 +frame_id: '' and grasped_object_mask header seq: 184495 +stamp: + secs: 1735698844 + nsecs: 557247639 +frame_id: "camera_color_optical_frame" +[INFO] [1735698845.086265]: received mask_obs with header seq: 2150 +stamp: + secs: 1735698845 + nsecs: 89871883 +frame_id: '' and grasped_object_mask header seq: 184504 +stamp: + secs: 1735698844 + nsecs: 857006073 +frame_id: "camera_color_optical_frame" +[INFO] [1735698845.335443]: received mask_obs with header seq: 2151 +stamp: + secs: 1735698845 + nsecs: 339085578 +frame_id: '' and grasped_object_mask header seq: 184511 +stamp: + secs: 1735698845 + nsecs: 90152025 +frame_id: "camera_color_optical_frame" +[INFO] [1735698845.596177]: received mask_obs with header seq: 2152 +stamp: + secs: 1735698845 + nsecs: 599837779 +frame_id: '' and grasped_object_mask header seq: 184519 +stamp: + secs: 1735698845 + nsecs: 357749701 +frame_id: "camera_color_optical_frame" +[INFO] [1735698845.852001]: received mask_obs with header seq: 2153 +stamp: + secs: 1735698845 + nsecs: 855631828 +frame_id: '' and grasped_object_mask header seq: 184527 +stamp: + secs: 1735698845 + nsecs: 627362967 +frame_id: "camera_color_optical_frame" +[INFO] [1735698846.120352]: received mask_obs with header seq: 2154 +stamp: + secs: 1735698846 + nsecs: 124011993 +frame_id: '' and grasped_object_mask header seq: 184535 +stamp: + secs: 1735698845 + nsecs: 897133112 +frame_id: "camera_color_optical_frame" +[INFO] [1735698846.382704]: received mask_obs with header seq: 2155 +stamp: + secs: 1735698846 + nsecs: 386306047 +frame_id: '' and grasped_object_mask header seq: 184543 +stamp: + secs: 1735698846 + nsecs: 167057514 +frame_id: "camera_color_optical_frame" +[INFO] [1735698846.648629]: received mask_obs with header seq: 2156 +stamp: + secs: 1735698846 + nsecs: 651803493 +frame_id: '' and grasped_object_mask header seq: 184551 +stamp: + secs: 1735698846 + nsecs: 434663773 +frame_id: "camera_color_optical_frame" +[INFO] [1735698846.914542]: received mask_obs with header seq: 2157 +stamp: + secs: 1735698846 + nsecs: 918146371 +frame_id: '' and grasped_object_mask header seq: 184558 +stamp: + secs: 1735698846 + nsecs: 667532682 +frame_id: "camera_color_optical_frame" +[INFO] [1735698847.183261]: received mask_obs with header seq: 2158 +stamp: + secs: 1735698847 + nsecs: 186693668 +frame_id: '' and grasped_object_mask header seq: 184567 +stamp: + secs: 1735698846 + nsecs: 966910362 +frame_id: "camera_color_optical_frame" +[INFO] [1735698847.446351]: received mask_obs with header seq: 2159 +stamp: + secs: 1735698847 + nsecs: 449979066 +frame_id: '' and grasped_object_mask header seq: 184575 +stamp: + secs: 1735698847 + nsecs: 233003139 +frame_id: "camera_color_optical_frame" +[INFO] [1735698847.697039]: received mask_obs with header seq: 2160 +stamp: + secs: 1735698847 + nsecs: 700332880 +frame_id: '' and grasped_object_mask header seq: 184582 +stamp: + secs: 1735698847 + nsecs: 465865135 +frame_id: "camera_color_optical_frame" +[INFO] [1735698847.958980]: received mask_obs with header seq: 2161 +stamp: + secs: 1735698847 + nsecs: 962313175 +frame_id: '' and grasped_object_mask header seq: 184590 +stamp: + secs: 1735698847 + nsecs: 731983900 +frame_id: "camera_color_optical_frame" +[INFO] [1735698848.215632]: received mask_obs with header seq: 2162 +stamp: + secs: 1735698848 + nsecs: 218672275 +frame_id: '' and grasped_object_mask header seq: 184598 +stamp: + secs: 1735698847 + nsecs: 998080015 +frame_id: "camera_color_optical_frame" +[INFO] [1735698848.481013]: received mask_obs with header seq: 2163 +stamp: + secs: 1735698848 + nsecs: 484115123 +frame_id: '' and grasped_object_mask header seq: 184606 +stamp: + secs: 1735698848 + nsecs: 264154673 +frame_id: "camera_color_optical_frame" +[INFO] [1735698848.745434]: received mask_obs with header seq: 2164 +stamp: + secs: 1735698848 + nsecs: 749020338 +frame_id: '' and grasped_object_mask header seq: 184613 +stamp: + secs: 1735698848 + nsecs: 497030735 +frame_id: "camera_color_optical_frame" +[INFO] [1735698848.995016]: received mask_obs with header seq: 2165 +stamp: + secs: 1735698848 + nsecs: 997522115 +frame_id: '' and grasped_object_mask header seq: 184621 +stamp: + secs: 1735698848 + nsecs: 763154745 +frame_id: "camera_color_optical_frame" +[INFO] [1735698849.251009]: received mask_obs with header seq: 2166 +stamp: + secs: 1735698849 + nsecs: 254654169 +frame_id: '' and grasped_object_mask header seq: 184629 +stamp: + secs: 1735698849 + nsecs: 29257536 +frame_id: "camera_color_optical_frame" +[INFO] [1735698849.514151]: received mask_obs with header seq: 2167 +stamp: + secs: 1735698849 + nsecs: 516956329 +frame_id: '' and grasped_object_mask header seq: 184637 +stamp: + secs: 1735698849 + nsecs: 295349598 +frame_id: "camera_color_optical_frame" +[INFO] [1735698849.768738]: received mask_obs with header seq: 2168 +stamp: + secs: 1735698849 + nsecs: 771224737 +frame_id: '' and grasped_object_mask header seq: 184644 +stamp: + secs: 1735698849 + nsecs: 528225899 +frame_id: "camera_color_optical_frame" +[INFO] [1735698850.025006]: received mask_obs with header seq: 2169 +stamp: + secs: 1735698850 + nsecs: 28645038 +frame_id: '' and grasped_object_mask header seq: 184652 +stamp: + secs: 1735698849 + nsecs: 794350147 +frame_id: "camera_color_optical_frame" +[INFO] [1735698850.282137]: received mask_obs with header seq: 2170 +stamp: + secs: 1735698850 + nsecs: 285825967 +frame_id: '' and grasped_object_mask header seq: 184660 +stamp: + secs: 1735698850 + nsecs: 60453892 +frame_id: "camera_color_optical_frame" +[INFO] [1735698850.546841]: received mask_obs with header seq: 2171 +stamp: + secs: 1735698850 + nsecs: 550258398 +frame_id: '' and grasped_object_mask header seq: 184668 +stamp: + secs: 1735698850 + nsecs: 326559305 +frame_id: "camera_color_optical_frame" +[INFO] [1735698850.811902]: received mask_obs with header seq: 2172 +stamp: + secs: 1735698850 + nsecs: 815576314 +frame_id: '' and grasped_object_mask header seq: 184676 +stamp: + secs: 1735698850 + nsecs: 592707157 +frame_id: "camera_color_optical_frame" +[INFO] [1735698851.062488]: received mask_obs with header seq: 2173 +stamp: + secs: 1735698851 + nsecs: 65588951 +frame_id: '' and grasped_object_mask header seq: 184683 +stamp: + secs: 1735698850 + nsecs: 825570345 +frame_id: "camera_color_optical_frame" +[INFO] [1735698851.330547]: received mask_obs with header seq: 2174 +stamp: + secs: 1735698851 + nsecs: 334187507 +frame_id: '' and grasped_object_mask header seq: 184691 +stamp: + secs: 1735698851 + nsecs: 91680050 +frame_id: "camera_color_optical_frame" +[INFO] [1735698851.579310]: received mask_obs with header seq: 2175 +stamp: + secs: 1735698851 + nsecs: 582757472 +frame_id: '' and grasped_object_mask header seq: 184699 +stamp: + secs: 1735698851 + nsecs: 357797861 +frame_id: "camera_color_optical_frame" +[INFO] [1735698851.839576]: received mask_obs with header seq: 2176 +stamp: + secs: 1735698851 + nsecs: 842766046 +frame_id: '' and grasped_object_mask header seq: 184707 +stamp: + secs: 1735698851 + nsecs: 623942614 +frame_id: "camera_color_optical_frame" +[INFO] [1735698852.092447]: received mask_obs with header seq: 2177 +stamp: + secs: 1735698852 + nsecs: 96088647 +frame_id: '' and grasped_object_mask header seq: 184714 +stamp: + secs: 1735698851 + nsecs: 856803179 +frame_id: "camera_color_optical_frame" +[INFO] [1735698852.353329]: received mask_obs with header seq: 2178 +stamp: + secs: 1735698852 + nsecs: 356907844 +frame_id: '' and grasped_object_mask header seq: 184722 +stamp: + secs: 1735698852 + nsecs: 122910023 +frame_id: "camera_color_optical_frame" +[INFO] [1735698852.613806]: received mask_obs with header seq: 2179 +stamp: + secs: 1735698852 + nsecs: 617446422 +frame_id: '' and grasped_object_mask header seq: 184730 +stamp: + secs: 1735698852 + nsecs: 389034033 +frame_id: "camera_color_optical_frame" +[INFO] [1735698852.860613]: received mask_obs with header seq: 2180 +stamp: + secs: 1735698852 + nsecs: 864108085 +frame_id: '' and grasped_object_mask header seq: 184737 +stamp: + secs: 1735698852 + nsecs: 621908903 +frame_id: "camera_color_optical_frame" +[INFO] [1735698853.122601]: received mask_obs with header seq: 2181 +stamp: + secs: 1735698853 + nsecs: 126044750 +frame_id: '' and grasped_object_mask header seq: 184745 +stamp: + secs: 1735698852 + nsecs: 888031960 +frame_id: "camera_color_optical_frame" +[INFO] [1735698853.381160]: received mask_obs with header seq: 2182 +stamp: + secs: 1735698853 + nsecs: 384565591 +frame_id: '' and grasped_object_mask header seq: 184753 +stamp: + secs: 1735698853 + nsecs: 154134274 +frame_id: "camera_color_optical_frame" +[INFO] [1735698853.644931]: received mask_obs with header seq: 2183 +stamp: + secs: 1735698853 + nsecs: 648134946 +frame_id: '' and grasped_object_mask header seq: 184761 +stamp: + secs: 1735698853 + nsecs: 420266151 +frame_id: "camera_color_optical_frame" +[INFO] [1735698853.896374]: received mask_obs with header seq: 2184 +stamp: + secs: 1735698853 + nsecs: 899987936 +frame_id: '' and grasped_object_mask header seq: 184768 +stamp: + secs: 1735698853 + nsecs: 653138876 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 100763953 for nearest: 1735698854020025409 - target: 1735698853919261456 at idx 249 +[INFO] [1735698854.153513]: received mask_obs with header seq: 2185 +stamp: + secs: 1735698854 + nsecs: 156686782 +frame_id: '' and grasped_object_mask header seq: 184776 +stamp: + secs: 1735698853 + nsecs: 919261456 +frame_id: "camera_color_optical_frame" +[INFO] [1735698854.411564]: received mask_obs with header seq: 2186 +stamp: + secs: 1735698854 + nsecs: 414858818 +frame_id: '' and grasped_object_mask header seq: 184784 +stamp: + secs: 1735698854 + nsecs: 185362577 +frame_id: "camera_color_optical_frame" +[INFO] [1735698854.675974]: received mask_obs with header seq: 2187 +stamp: + secs: 1735698854 + nsecs: 677556991 +frame_id: '' and grasped_object_mask header seq: 184792 +stamp: + secs: 1735698854 + nsecs: 451514482 +frame_id: "camera_color_optical_frame" +[INFO] [1735698854.936400]: received mask_obs with header seq: 2188 +stamp: + secs: 1735698854 + nsecs: 938856363 +frame_id: '' and grasped_object_mask header seq: 184799 +stamp: + secs: 1735698854 + nsecs: 684375763 +frame_id: "camera_color_optical_frame" +[INFO] [1735698855.186234]: received mask_obs with header seq: 2189 +stamp: + secs: 1735698855 + nsecs: 189500331 +frame_id: '' and grasped_object_mask header seq: 184807 +stamp: + secs: 1735698854 + nsecs: 950482130 +frame_id: "camera_color_optical_frame" +[INFO] [1735698855.442788]: received mask_obs with header seq: 2190 +stamp: + secs: 1735698855 + nsecs: 446327924 +frame_id: '' and grasped_object_mask header seq: 184815 +stamp: + secs: 1735698855 + nsecs: 216567039 +frame_id: "camera_color_optical_frame" +[INFO] [1735698855.708145]: received mask_obs with header seq: 2191 +stamp: + secs: 1735698855 + nsecs: 711070060 +frame_id: '' and grasped_object_mask header seq: 184823 +stamp: + secs: 1735698855 + nsecs: 482702494 +frame_id: "camera_color_optical_frame" +[INFO] [1735698855.972984]: received mask_obs with header seq: 2192 +stamp: + secs: 1735698855 + nsecs: 976597070 +frame_id: '' and grasped_object_mask header seq: 184831 +stamp: + secs: 1735698855 + nsecs: 748838663 +frame_id: "camera_color_optical_frame" +[INFO] [1735698856.230256]: received mask_obs with header seq: 2193 +stamp: + secs: 1735698856 + nsecs: 233347415 +frame_id: '' and grasped_object_mask header seq: 184838 +stamp: + secs: 1735698855 + nsecs: 981689215 +frame_id: "camera_color_optical_frame" +[INFO] [1735698856.491034]: received mask_obs with header seq: 2194 +stamp: + secs: 1735698856 + nsecs: 494590759 +frame_id: '' and grasped_object_mask header seq: 184846 +stamp: + secs: 1735698856 + nsecs: 247786283 +frame_id: "camera_color_optical_frame" +[INFO] [1735698856.749106]: received mask_obs with header seq: 2195 +stamp: + secs: 1735698856 + nsecs: 752679347 +frame_id: '' and grasped_object_mask header seq: 184854 +stamp: + secs: 1735698856 + nsecs: 513931513 +frame_id: "camera_color_optical_frame" +[INFO] [1735698857.006891]: received mask_obs with header seq: 2196 +stamp: + secs: 1735698857 + nsecs: 9990692 +frame_id: '' and grasped_object_mask header seq: 184862 +stamp: + secs: 1735698856 + nsecs: 780063629 +frame_id: "camera_color_optical_frame" +[INFO] [1735698857.270904]: received mask_obs with header seq: 2197 +stamp: + secs: 1735698857 + nsecs: 274364471 +frame_id: '' and grasped_object_mask header seq: 184870 +stamp: + secs: 1735698857 + nsecs: 46175241 +frame_id: "camera_color_optical_frame" +[INFO] [1735698857.535504]: received mask_obs with header seq: 2198 +stamp: + secs: 1735698857 + nsecs: 538693189 +frame_id: '' and grasped_object_mask header seq: 184877 +stamp: + secs: 1735698857 + nsecs: 279007912 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 108161743 for nearest: 1735698857653311546 - target: 1735698857545149803 at idx 249 +[INFO] [1735698857.790480]: received mask_obs with header seq: 2199 +stamp: + secs: 1735698857 + nsecs: 794146060 +frame_id: '' and grasped_object_mask header seq: 184885 +stamp: + secs: 1735698857 + nsecs: 545149803 +frame_id: "camera_color_optical_frame" +[INFO] [1735698858.047467]: received mask_obs with header seq: 2200 +stamp: + secs: 1735698858 + nsecs: 51069021 +frame_id: '' and grasped_object_mask header seq: 184893 +stamp: + secs: 1735698857 + nsecs: 811270952 +frame_id: "camera_color_optical_frame" +[INFO] [1735698858.314370]: received mask_obs with header seq: 2201 +stamp: + secs: 1735698858 + nsecs: 317638397 +frame_id: '' and grasped_object_mask header seq: 184901 +stamp: + secs: 1735698858 + nsecs: 77370644 +frame_id: "camera_color_optical_frame" +[INFO] [1735698858.570581]: received mask_obs with header seq: 2202 +stamp: + secs: 1735698858 + nsecs: 573706865 +frame_id: '' and grasped_object_mask header seq: 184909 +stamp: + secs: 1735698858 + nsecs: 343473673 +frame_id: "camera_color_optical_frame" +[INFO] [1735698858.817363]: received mask_obs with header seq: 2203 +stamp: + secs: 1735698858 + nsecs: 820855617 +frame_id: '' and grasped_object_mask header seq: 184916 +stamp: + secs: 1735698858 + nsecs: 576351166 +frame_id: "camera_color_optical_frame" +[INFO] [1735698859.075883]: received mask_obs with header seq: 2204 +stamp: + secs: 1735698859 + nsecs: 79102277 +frame_id: '' and grasped_object_mask header seq: 184924 +stamp: + secs: 1735698858 + nsecs: 842479229 +frame_id: "camera_color_optical_frame" +[INFO] [1735698859.339445]: received mask_obs with header seq: 2205 +stamp: + secs: 1735698859 + nsecs: 343102931 +frame_id: '' and grasped_object_mask header seq: 184932 +stamp: + secs: 1735698859 + nsecs: 108585358 +frame_id: "camera_color_optical_frame" +[INFO] [1735698859.602798]: received mask_obs with header seq: 2206 +stamp: + secs: 1735698859 + nsecs: 606384038 +frame_id: '' and grasped_object_mask header seq: 184939 +stamp: + secs: 1735698859 + nsecs: 341433287 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 112391407 for nearest: 1735698859719972307 - target: 1735698859607580900 at idx 249 +[INFO] [1735698859.851100]: received mask_obs with header seq: 2207 +stamp: + secs: 1735698859 + nsecs: 854707956 +frame_id: '' and grasped_object_mask header seq: 184947 +stamp: + secs: 1735698859 + nsecs: 607580900 +frame_id: "camera_color_optical_frame" +[INFO] [1735698860.112488]: received mask_obs with header seq: 2208 +stamp: + secs: 1735698860 + nsecs: 116021871 +frame_id: '' and grasped_object_mask header seq: 184955 +stamp: + secs: 1735698859 + nsecs: 873707771 +frame_id: "camera_color_optical_frame" +[INFO] [1735698860.371248]: received mask_obs with header seq: 2209 +stamp: + secs: 1735698860 + nsecs: 374620437 +frame_id: '' and grasped_object_mask header seq: 184963 +stamp: + secs: 1735698860 + nsecs: 139813900 +frame_id: "camera_color_optical_frame" +[INFO] [1735698860.637345]: received mask_obs with header seq: 2210 +stamp: + secs: 1735698860 + nsecs: 640830516 +frame_id: '' and grasped_object_mask header seq: 184971 +stamp: + secs: 1735698860 + nsecs: 405935764 +frame_id: "camera_color_optical_frame" +[INFO] [1735698860.893510]: received mask_obs with header seq: 2211 +stamp: + secs: 1735698860 + nsecs: 895478963 +frame_id: '' and grasped_object_mask header seq: 184978 +stamp: + secs: 1735698860 + nsecs: 638805628 +frame_id: "camera_color_optical_frame" +[INFO] [1735698861.151148]: received mask_obs with header seq: 2212 +stamp: + secs: 1735698861 + nsecs: 154371500 +frame_id: '' and grasped_object_mask header seq: 184986 +stamp: + secs: 1735698860 + nsecs: 904922247 +frame_id: "camera_color_optical_frame" +[INFO] [1735698861.408363]: received mask_obs with header seq: 2213 +stamp: + secs: 1735698861 + nsecs: 411499977 +frame_id: '' and grasped_object_mask header seq: 184994 +stamp: + secs: 1735698861 + nsecs: 171018124 +frame_id: "camera_color_optical_frame" +[INFO] [1735698861.669501]: received mask_obs with header seq: 2214 +stamp: + secs: 1735698861 + nsecs: 672747611 +frame_id: '' and grasped_object_mask header seq: 185002 +stamp: + secs: 1735698861 + nsecs: 438239336 +frame_id: "camera_color_optical_frame" +[INFO] [1735698861.936726]: received mask_obs with header seq: 2215 +stamp: + secs: 1735698861 + nsecs: 940292596 +frame_id: '' and grasped_object_mask header seq: 185010 +stamp: + secs: 1735698861 + nsecs: 706240416 +frame_id: "camera_color_optical_frame" +[INFO] [1735698862.202224]: received mask_obs with header seq: 2216 +stamp: + secs: 1735698862 + nsecs: 205475807 +frame_id: '' and grasped_object_mask header seq: 185018 +stamp: + secs: 1735698861 + nsecs: 974398136 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 110857874 for nearest: 1735698862320023686 - target: 1735698862209165812 at idx 249 +[INFO] [1735698862.456141]: received mask_obs with header seq: 2217 +stamp: + secs: 1735698862 + nsecs: 459814310 +frame_id: '' and grasped_object_mask header seq: 185025 +stamp: + secs: 1735698862 + nsecs: 209165812 +frame_id: "camera_color_optical_frame" +[INFO] [1735698862.712528]: received mask_obs with header seq: 2218 +stamp: + secs: 1735698862 + nsecs: 716126918 +frame_id: '' and grasped_object_mask header seq: 185033 +stamp: + secs: 1735698862 + nsecs: 476091385 +frame_id: "camera_color_optical_frame" +[INFO] [1735698862.973608]: received mask_obs with header seq: 2219 +stamp: + secs: 1735698862 + nsecs: 977295637 +frame_id: '' and grasped_object_mask header seq: 185041 +stamp: + secs: 1735698862 + nsecs: 742551088 +frame_id: "camera_color_optical_frame" +[INFO] [1735698863.236428]: received mask_obs with header seq: 2220 +stamp: + secs: 1735698863 + nsecs: 239612102 +frame_id: '' and grasped_object_mask header seq: 185049 +stamp: + secs: 1735698863 + nsecs: 9011507 +frame_id: "camera_color_optical_frame" +[INFO] [1735698863.501979]: received mask_obs with header seq: 2221 +stamp: + secs: 1735698863 + nsecs: 505204439 +frame_id: '' and grasped_object_mask header seq: 185057 +stamp: + secs: 1735698863 + nsecs: 275473356 +frame_id: "camera_color_optical_frame" +[INFO] [1735698863.766023]: received mask_obs with header seq: 2222 +stamp: + secs: 1735698863 + nsecs: 769631147 +frame_id: '' and grasped_object_mask header seq: 185064 +stamp: + secs: 1735698863 + nsecs: 508609295 +frame_id: "camera_color_optical_frame" +[INFO] [1735698864.036146]: received mask_obs with header seq: 2223 +stamp: + secs: 1735698864 + nsecs: 39073228 +frame_id: '' and grasped_object_mask header seq: 185072 +stamp: + secs: 1735698863 + nsecs: 775051594 +frame_id: "camera_color_optical_frame" +[INFO] [1735698864.303488]: received mask_obs with header seq: 2224 +stamp: + secs: 1735698864 + nsecs: 306460142 +frame_id: '' and grasped_object_mask header seq: 185080 +stamp: + secs: 1735698864 + nsecs: 41491985 +frame_id: "camera_color_optical_frame" +[INFO] [1735698864.571306]: received mask_obs with header seq: 2225 +stamp: + secs: 1735698864 + nsecs: 574925899 +frame_id: '' and grasped_object_mask header seq: 185089 +stamp: + secs: 1735698864 + nsecs: 341239691 +frame_id: "camera_color_optical_frame" +[INFO] [1735698864.679204]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1735698864.837616]: received mask_obs with header seq: 2226 +stamp: + secs: 1735698864 + nsecs: 838769674 +frame_id: '' and grasped_object_mask header seq: 185097 +stamp: + secs: 1735698864 + nsecs: 607689142 +frame_id: "camera_color_optical_frame" +[INFO] [1735698865.421960]: Stored episode 5. +[INFO] [1735698865.428707]: Saving video... +[INFO] [1735698865.641663]: Video saved! +[INFO] [1735698865.691741]: resetting environment +[INFO] [1735698865.708338]: cleared current plan +[INFO] [1735698865.708559]: moving to home +[INFO] [1735698869.320635]: reached home +[INFO] [1735698869.328184]: reset action history +[INFO] [1735698872.146360]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/requirements.txt b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-metadata.json b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..2b8af73e89fda3a401908d913276aec452969087 --- /dev/null +++ b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T02:27:31.971767", + "startedAt": "2025-01-01T02:27:31.561866", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3928.982, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.92, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4494.556, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4469.135, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 507.49398040771484 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..94139b5f088acb0ac73cb59494c6ecf2bb038f73 --- /dev/null +++ b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "73394dae2cb5c0e3e76ddd01183d5fb1de9dd95392128b29d1c428faa8a3c842", "size": 706666, "path": "media/videos/eval/0_eval_0_73394dae2cb5c0e3e76d.mp4"}, "global_step": 428, "_timestamp": 1735698876.400416, "_runtime": 424.82818388938904, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "eb820ba01fc256486a6ce64bb8e09317279dd63b1bb04cdd34a8f99ac85e8747", "size": 621660, "path": "media/videos/eval/1_eval_1_eb820ba01fc256486a6c.mp4"}, "eval/num_success": 2.0, "episode": 5.0, "eval/success_rate": 0.4000000059604645, "eval/success_rate_lb_0.01": 0.044009119272232056, "eval/success_rate_ub_0.01": 0.8621948957443237, "eval/success_rate_lb_0.025": 0.05324718356132507, "eval/success_rate_ub_0.025": 0.803534984588623, "eval/success_rate_lb_0.05": 0.11712844669818878, "eval/success_rate_ub_0.05": 0.7487166523933411, "eval/success_rate_lb_0.1": 0.23282015323638916, "eval/success_rate_ub_0.1": 0.6637700200080872, "eval/2_eval": {"_type": "video-file", "sha256": "8637db17a31d8f9e9a53089018168b3d8b4ac0608ff98f962c66d4250bf80a3f", "size": 689867, "path": "media/videos/eval/2_eval_2_8637db17a31d8f9e9a53.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "4f1bea0839448d87829bcde0ef5cbba8ece5cbb3d99968a605f93e85d36e44ce", "size": 1566098, "path": "media/videos/eval/3_eval_3_4f1bea0839448d87829b.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "4b071c579be3fd81d6bdd5c557c3c3e485a6e0d9f3e8e02f49c17a533ec3a014", "size": 1534428, "path": "media/videos/eval/4_eval_4_4b071c579be3fd81d6bd.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "85c54431b4eb7c8145a72a6437cbffccb84a9cf38051f18b07dd0a638d459f25", "size": 401, "artifact_path": "wandb-client-artifact://rxqcuide80w82kyplqhy91j3n167wav966sp3yxgzdil9y87372hpkwzdr9l9y41w76tnp766hf858j5rinjmm5h2u9cszg48kmmg034lh05c7ztbxhzy8tidv0x0ta5/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://gcsftdkes89hukq9l1ggraic4l4vt0uu9djj6gici98ot0mmvgahdkja4swgw8wfacjdtojapm5zl2xbtpcakifvwfrrpqlc9crtb7u6rh0j8loiv2t2srw6v5rmtund:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 2, "total_num_episodes": 5, "total_success_rate": 0.4, "total_success_rate_lb_0.01": 0.04070010790474144, "total_success_rate_ub_0.01": 0.8728086947649158, "total_success_rate_lb_0.025": 0.09036684044447546, "total_success_rate_ub_0.025": 0.8503108768954606, "total_success_rate_lb_0.05": 0.12382420905007341, "total_success_rate_ub_0.05": 0.7396230101106143, "total_success_rate_lb_0.1": 0.1259237081560607, "total_success_rate_ub_0.1": 0.7419417798035252, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "d5fa38cba1266bd8cfebdb65df17fa0d0fd3a3a64e30f6512f69c2b67c0ef0fc", "size": 506, "artifact_path": "wandb-client-artifact://g8wh2al2jt8pzwyw0viju7la12gp6w11hmfgdwt7nn69l8j328ufbhe3h6dy5h6ukaw2ey1fuw9znlgr40alnq09mttw5qrw9c146jt6iy3iju8kl0pbqdn0cm4u7zvc/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://qtc50alhiwabnc9lymm9m3mywntxc2wvr8m765ou3cfsxmr4020glk3y2z7p698u9ve63eqvs7d59p2b1oyqii35xdi8me2iuhf0duhe8f8m6x75el220k9jo3qep622:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 425}, "eval/total_success_rate": 0.4000000059604645, "eval/total_success_rate_lb_0.01": 0.04070010781288147, "eval/total_success_rate_ub_0.01": 0.8728086948394775, "eval/total_success_rate_lb_0.025": 0.09036684036254883, "eval/total_success_rate_ub_0.025": 0.8503108620643616, "eval/total_success_rate_lb_0.05": 0.12382420897483826, "eval/total_success_rate_ub_0.05": 0.7396230101585388, "eval/total_success_rate_lb_0.1": 0.12592370808124542, "eval/total_success_rate_ub_0.1": 0.7419417500495911} \ No newline at end of file diff --git a/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/logs/debug-internal.log b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..df078523addccacbe9cc5f318604cc24abdc2631 --- /dev/null +++ b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/logs/debug-internal.log @@ -0,0 +1,1034 @@ +2024-12-31 21:27:31,572 INFO StreamThr :1746604 [internal.py:wandb_internal():85] W&B internal server running at pid: 1746604, started at: 2024-12-31 21:27:31.571700 +2024-12-31 21:27:31,573 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status +2024-12-31 21:27:31,584 INFO WriterThread:1746604 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/run-z0y7vp39.wandb +2024-12-31 21:27:31,584 DEBUG SenderThread:1746604 [sender.py:send():379] send: header +2024-12-31 21:27:31,592 DEBUG SenderThread:1746604 [sender.py:send():379] send: run +2024-12-31 21:27:31,853 INFO SenderThread:1746604 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files +2024-12-31 21:27:31,853 INFO SenderThread:1746604 [sender.py:_start_run_threads():1188] run started: z0y7vp39 with start time 1735698451.572232 +2024-12-31 21:27:31,870 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 21:27:31,870 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: check_version +2024-12-31 21:27:31,953 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 21:27:31,958 DEBUG HandlerThread:1746604 [system_info.py:__init__():26] System info init +2024-12-31 21:27:31,958 DEBUG HandlerThread:1746604 [system_info.py:__init__():41] System info init done +2024-12-31 21:27:31,958 INFO HandlerThread:1746604 [system_monitor.py:start():194] Starting system monitor +2024-12-31 21:27:31,958 INFO SystemMonitor:1746604 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 21:27:31,958 INFO HandlerThread:1746604 [system_monitor.py:probe():214] Collecting system info +2024-12-31 21:27:31,959 INFO SystemMonitor:1746604 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 21:27:31,959 INFO SystemMonitor:1746604 [interfaces.py:start():188] Started disk monitoring +2024-12-31 21:27:31,960 INFO SystemMonitor:1746604 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 21:27:31,960 INFO SystemMonitor:1746604 [interfaces.py:start():188] Started memory monitoring +2024-12-31 21:27:31,961 INFO SystemMonitor:1746604 [interfaces.py:start():188] Started network monitoring +2024-12-31 21:27:31,971 DEBUG HandlerThread:1746604 [system_info.py:probe():152] Probing system +2024-12-31 21:27:31,973 DEBUG HandlerThread:1746604 [system_info.py:_probe_git():137] Probing git +2024-12-31 21:27:31,977 DEBUG HandlerThread:1746604 [system_info.py:_probe_git():145] Probing git done +2024-12-31 21:27:31,977 DEBUG HandlerThread:1746604 [system_info.py:probe():200] Probing system done +2024-12-31 21:27:31,977 DEBUG HandlerThread:1746604 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T02:27:31.971767', 'startedAt': '2025-01-01T02:27:31.561866', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3928.982, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.92, 'min': 800.0, 'max': 4700.0}, {'current': 4494.556, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4469.135, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 507.49398040771484}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 21:27:31,978 INFO HandlerThread:1746604 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 21:27:31,978 INFO HandlerThread:1746604 [system_monitor.py:probe():227] Publishing system info +2024-12-31 21:27:31,978 DEBUG HandlerThread:1746604 [system_info.py:_save_code():44] Saving code +2024-12-31 21:27:31,979 DEBUG HandlerThread:1746604 [system_info.py:_save_code():67] Saving code done +2024-12-31 21:27:31,979 DEBUG HandlerThread:1746604 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 21:27:32,042 DEBUG HandlerThread:1746604 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 21:27:32,042 INFO HandlerThread:1746604 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 21:27:32,045 DEBUG SenderThread:1746604 [sender.py:send():379] send: files +2024-12-31 21:27:32,045 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 21:27:32,045 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 21:27:32,046 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 21:27:32,277 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/s4vfpuwy-wandb-metadata.json +2024-12-31 21:27:32,282 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 21:27:32,282 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: python_packages +2024-12-31 21:27:32,284 DEBUG SenderThread:1746604 [sender.py:send():379] send: telemetry +2024-12-31 21:27:32,320 INFO wandb-upload_1:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/cwia0lzj-code/FISH/eval_robot.py +2024-12-31 21:27:32,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:32,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:27:32,325 INFO HandlerThread:1746604 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/tb" + save: true +} +control { + relay_id: "c837d0f9c03b46dc9ed679278affa699" +} +_info { + stream_id: "z0y7vp39" +} + +2024-12-31 21:27:32,325 DEBUG HandlerThread:1746604 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 21:27:32,326 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:27:32,344 INFO wandb-upload_2:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/cfcnmz7m-diff.patch +2024-12-31 21:27:32,398 DEBUG SenderThread:1746604 [sender.py:send():379] send: tbrecord +2024-12-31 21:27:32,399 DEBUG SenderThread:1746604 [sender.py:send():379] send: files +2024-12-31 21:27:32,399 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 with policy live +2024-12-31 21:27:32,579 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/yjw00hgk-events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:27:32,855 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:27:32,855 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/requirements.txt +2024-12-31 21:27:32,855 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/code/FISH/eval_robot.py +2024-12-31 21:27:32,855 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/diff.patch +2024-12-31 21:27:32,855 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:27:32,855 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-metadata.json +2024-12-31 21:27:32,855 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/code +2024-12-31 21:27:32,855 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/code/FISH +2024-12-31 21:27:33,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:34,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:34,854 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:27:35,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:36,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:37,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:27:37,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:38,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:39,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:40,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:41,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:42,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:27:42,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:43,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:44,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:45,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:46,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:47,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:27:47,283 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:27:47,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:47,434 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:27:48,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:49,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:50,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:51,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:52,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:53,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:27:53,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:54,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:55,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:56,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:57,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:58,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:27:58,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:27:59,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:00,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:01,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:02,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:28:02,283 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:28:02,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:03,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:03,436 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:03,861 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/config.yaml +2024-12-31 21:28:04,287 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:05,287 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:06,287 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:07,287 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:08,287 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:09,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:09,287 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:10,287 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:11,287 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:12,287 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:13,288 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:14,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:14,288 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:15,288 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:16,288 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:17,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:28:17,283 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:28:17,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:18,289 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:19,289 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:19,367 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:20,289 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:21,290 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:22,290 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:23,290 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:24,290 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:25,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:25,290 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:26,290 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:27,245 DEBUG SenderThread:1746604 [sender.py:send():379] send: config +2024-12-31 21:28:27,290 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:28,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:28,868 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:28:29,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:30,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:30,298 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:30,868 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:28:31,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:31,961 DEBUG SystemMonitor:1746604 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 21:28:31,962 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:28:32,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:28:32,283 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:28:32,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:32,869 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:28:33,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:34,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:34,870 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:28:35,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:35,789 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:36,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:36,892 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:28:36,893 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/config.yaml +2024-12-31 21:28:37,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:38,292 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:39,292 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:40,292 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:41,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:41,292 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:42,292 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:43,292 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:44,292 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:45,293 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:46,293 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:47,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:28:47,283 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:28:47,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:47,342 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:48,294 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:49,294 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:50,294 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:51,294 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:52,294 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:53,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:53,294 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:54,295 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:55,295 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:56,295 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:57,295 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:58,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:28:58,295 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:28:59,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:00,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:01,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:01,962 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:29:02,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:29:02,283 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:29:02,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:03,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:03,344 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:04,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:05,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:06,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:07,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:08,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:09,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:09,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:10,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:11,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:12,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:13,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:14,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:14,298 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:15,298 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:16,298 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:17,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:29:17,284 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:29:17,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:18,298 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:18,904 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:19,299 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:19,437 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:20,299 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:20,904 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:21,301 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:22,299 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:22,905 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:23,299 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:24,302 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:24,504 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:24,906 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:25,301 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:26,301 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:26,906 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:27,301 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:28,301 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:28,907 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:29,301 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:29,709 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:30,301 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:30,908 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:31,301 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:31,964 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:29:32,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:29:32,284 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:29:32,302 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:32,908 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:33,302 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:34,303 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:34,891 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:34,909 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:35,302 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:36,302 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:36,909 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:37,303 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:37,546 DEBUG SenderThread:1746604 [sender.py:send():379] send: files +2024-12-31 21:29:37,547 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_73394dae2cb5c0e3e76d.mp4 with policy now +2024-12-31 21:29:37,547 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:29:37,548 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:29:37,548 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:29:37,548 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:29:37,792 INFO wandb-upload_2:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/1gw6jqwc-media/videos/eval/0_eval_0_73394dae2cb5c0e3e76d.mp4 +2024-12-31 21:29:37,910 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/0_eval_0_73394dae2cb5c0e3e76d.mp4 +2024-12-31 21:29:37,910 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:29:37,910 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval +2024-12-31 21:29:37,910 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media +2024-12-31 21:29:37,910 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos +2024-12-31 21:29:38,159 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/ic7jcpja-media/videos/eval/0_eval_0_73394dae2cb5c0e3e76d.mp4 +2024-12-31 21:29:38,303 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:38,910 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:39,303 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:40,303 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:40,558 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:41,303 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:42,303 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:42,912 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:43,303 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:44,304 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:44,912 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:45,304 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:46,304 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:46,518 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:47,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:29:47,284 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:29:47,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:48,305 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:49,305 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:50,306 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:51,305 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:52,290 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:52,305 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:53,305 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:54,305 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:54,916 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:29:55,179 INFO wandb-upload_2:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/wky0b0d2-events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:29:55,306 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:56,305 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:56,916 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:29:57,306 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:58,241 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:29:58,306 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:29:59,306 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:00,306 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:00,918 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:01,306 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:01,964 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:30:02,283 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:30:02,284 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:30:02,306 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:02,918 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:03,307 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:03,417 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:04,307 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:04,919 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:05,308 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:06,307 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:06,920 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:07,308 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:08,308 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:08,637 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:08,921 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:09,308 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:10,308 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:10,921 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:11,308 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:12,316 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:12,751 DEBUG SenderThread:1746604 [sender.py:send():379] send: files +2024-12-31 21:30:12,751 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_eb820ba01fc256486a6c.mp4 with policy now +2024-12-31 21:30:12,755 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:30:12,755 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:30:12,755 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:30:12,756 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:30:12,922 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:12,922 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:30:12,922 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/1_eval_1_eb820ba01fc256486a6c.mp4 +2024-12-31 21:30:12,923 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval +2024-12-31 21:30:12,986 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/n1i42p9m-media/videos/eval/1_eval_1_eb820ba01fc256486a6c.mp4 +2024-12-31 21:30:13,238 INFO wandb-upload_2:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/jfr6ugmk-media/videos/eval/1_eval_1_eb820ba01fc256486a6c.mp4 +2024-12-31 21:30:13,311 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:13,772 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:14,310 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:14,923 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:15,314 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:16,318 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:16,923 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:17,291 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:30:17,291 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:30:17,333 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:18,312 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:18,862 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:19,310 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:20,313 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:20,925 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:21,313 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:22,314 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:23,313 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:24,295 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:24,320 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:25,319 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:26,315 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:27,319 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:28,317 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:29,326 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:30,295 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:30,319 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:31,319 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:31,965 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:30:32,285 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:30:32,285 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:30:32,318 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:32,841 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:30:32,841 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:30:32,842 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:30:32,928 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:30:32,929 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:30:33,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:34,319 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:34,929 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:35,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:35,321 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:36,322 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:36,930 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:37,345 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:38,322 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:38,931 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:39,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:40,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:40,363 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:40,932 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:41,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:42,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:42,932 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:43,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:44,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:44,933 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:45,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:45,567 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:46,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:46,934 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:47,284 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:30:47,285 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:30:47,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:48,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:48,934 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:49,323 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:50,324 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:50,784 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:50,935 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:51,333 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:51,909 DEBUG SenderThread:1746604 [sender.py:send():379] send: files +2024-12-31 21:30:51,909 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_8637db17a31d8f9e9a53.mp4 with policy now +2024-12-31 21:30:51,913 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:30:51,913 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:30:51,913 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:30:51,913 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:30:51,936 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:30:51,936 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/2_eval_2_8637db17a31d8f9e9a53.mp4 +2024-12-31 21:30:51,936 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval +2024-12-31 21:30:52,147 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/jx77f427-media/videos/eval/2_eval_2_8637db17a31d8f9e9a53.mp4 +2024-12-31 21:30:52,324 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:52,398 INFO wandb-upload_2:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/fp0x8fjc-media/videos/eval/2_eval_2_8637db17a31d8f9e9a53.mp4 +2024-12-31 21:30:52,936 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:53,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:54,330 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:55,328 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:56,189 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:30:56,333 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:56,937 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:57,328 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:58,327 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:30:58,938 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:30:59,334 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:00,333 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:01,331 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:01,965 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:31:01,966 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:02,289 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:31:02,289 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:31:02,344 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:03,344 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:04,339 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:05,343 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:06,342 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:07,304 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:07,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:08,343 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:09,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:09,646 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:31:09,646 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:31:09,647 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:31:09,941 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:31:09,941 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:31:10,176 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/3lslso2z-events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:31:10,343 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:10,942 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:11,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:12,343 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:12,919 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:13,343 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:14,345 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:14,943 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:15,343 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:16,343 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:16,944 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:17,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:31:17,286 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:31:17,343 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:17,935 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:18,344 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:18,944 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:19,344 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:20,345 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:20,945 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:21,345 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:22,345 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:22,946 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:23,151 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:23,345 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:24,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:24,947 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:25,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:26,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:26,947 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:27,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:28,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:28,395 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:28,948 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:29,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:30,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:30,949 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:31,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:31,966 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:31:32,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:31:32,287 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:31:32,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:32,950 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:33,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:33,646 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:34,346 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:34,950 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:35,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:36,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:36,951 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:37,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:38,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:38,873 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:38,952 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:39,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:40,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:40,953 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:41,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:42,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:42,953 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:43,347 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:44,089 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:44,348 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:44,954 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:45,348 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:46,348 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:46,955 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:47,286 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:31:47,286 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:31:47,348 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:48,348 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:48,956 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:49,251 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:49,348 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:50,351 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:50,422 DEBUG SenderThread:1746604 [sender.py:send():379] send: files +2024-12-31 21:31:50,422 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4f1bea0839448d87829b.mp4 with policy now +2024-12-31 21:31:50,423 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:31:50,423 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:31:50,424 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:31:50,432 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:31:50,674 INFO wandb-upload_2:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/rippbbpg-media/videos/eval/3_eval_3_4f1bea0839448d87829b.mp4 +2024-12-31 21:31:50,956 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:50,956 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:31:50,957 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/3_eval_3_4f1bea0839448d87829b.mp4 +2024-12-31 21:31:50,957 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval +2024-12-31 21:31:51,228 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/v41wqpra-media/videos/eval/3_eval_3_4f1bea0839448d87829b.mp4 +2024-12-31 21:31:51,351 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:52,354 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:52,957 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:53,361 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:54,356 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:54,958 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:55,105 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:31:55,358 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:56,354 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:57,354 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:58,361 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:31:58,959 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:31:59,361 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:00,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:00,365 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:01,372 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:01,966 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:32:02,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:32:02,297 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:32:02,364 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:03,367 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:04,374 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:05,374 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:05,381 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:06,372 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:07,371 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:08,382 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:09,368 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:10,376 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:11,300 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:11,377 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:12,374 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:13,380 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:14,382 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:15,382 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:16,304 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:16,387 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:17,298 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:32:17,299 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:32:17,379 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:18,390 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:19,389 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:20,387 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:21,381 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:21,440 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:22,390 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:23,385 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:24,389 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:25,391 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:26,389 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:27,302 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:27,390 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:28,390 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:29,389 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:30,393 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:31,401 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:31,966 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:32:32,298 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:32:32,298 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:32:32,365 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:32,413 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:33,414 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:34,408 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:35,412 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:36,408 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:37,417 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:38,309 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:38,420 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:39,418 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:40,418 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:41,420 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:42,423 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:43,420 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:44,307 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:44,421 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:45,436 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:46,429 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:47,297 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:32:47,297 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:32:47,421 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:48,427 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:49,436 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:49,437 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:50,432 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:51,435 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:52,436 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:53,434 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:54,438 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:55,310 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:32:55,444 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:56,439 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:57,439 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:58,457 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:32:59,446 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:00,449 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:01,308 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:01,456 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:01,967 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:33:02,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:33:02,296 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:33:02,456 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:03,454 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:04,457 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:05,456 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:06,369 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:06,458 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:07,459 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:08,464 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:09,460 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:10,466 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:11,465 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:12,309 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:12,477 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:13,466 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:14,471 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:15,471 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:16,491 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:17,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:33:17,296 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:33:17,408 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:17,478 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:18,476 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:19,477 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:20,491 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:21,482 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:22,479 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:23,316 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:23,489 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:24,481 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:25,483 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:26,485 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:27,489 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:28,327 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:28,493 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:29,500 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:30,489 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:31,491 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:31,967 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:33:32,299 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:33:32,299 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:33:32,494 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:33,423 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:33,501 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:34,493 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:35,495 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:36,495 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:37,503 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:38,499 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:39,325 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:39,503 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:40,502 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:41,500 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:42,501 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:43,505 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:43,822 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:33:43,822 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:33:43,823 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:33:43,994 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:33:43,994 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:33:44,229 INFO wandb-upload_2:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/56q81p6h-events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:33:44,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:44,878 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:44,994 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:33:45,511 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:46,505 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:46,995 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:33:47,301 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:33:47,301 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:33:47,510 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:48,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:48,995 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:33:49,511 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:50,006 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:50,504 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:50,996 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:33:51,504 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:52,504 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:52,997 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:33:53,504 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:54,504 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:54,998 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:33:55,168 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:33:55,505 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:56,505 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:56,998 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:33:57,549 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:58,505 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:33:58,999 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:33:59,505 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:00,355 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:00,505 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:01,000 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:01,505 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:01,968 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:34:02,295 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:34:02,296 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:34:02,506 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:03,001 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:03,508 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:04,506 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:05,001 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:05,506 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:05,596 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:06,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:07,002 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:07,506 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:08,506 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:09,003 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:09,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:10,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:10,812 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:11,004 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:11,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:12,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:13,005 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:13,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:14,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:15,005 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:15,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:15,973 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:16,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:17,006 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:17,296 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:34:17,296 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:34:17,507 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:18,508 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:19,007 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:19,508 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:20,508 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:21,008 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:21,151 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:21,508 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:22,508 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:23,009 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:23,508 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:24,508 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:25,009 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:25,546 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:25,689 DEBUG SenderThread:1746604 [sender.py:send():379] send: files +2024-12-31 21:34:25,689 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_4b071c579be3fd81d6bd.mp4 with policy now +2024-12-31 21:34:25,692 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:34:25,692 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:34:25,692 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:25,693 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:25,940 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/oh9x4g5d-media/videos/eval/4_eval_4_4b071c579be3fd81d6bd.mp4 +2024-12-31 21:34:26,010 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:34:26,010 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/4_eval_4_4b071c579be3fd81d6bd.mp4 +2024-12-31 21:34:26,010 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval +2024-12-31 21:34:26,269 INFO wandb-upload_2:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/8zrrmdp2-media/videos/eval/4_eval_4_4b071c579be3fd81d6bd.mp4 +2024-12-31 21:34:26,533 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:26,709 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:27,010 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:27,537 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:28,538 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:29,538 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:30,539 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:31,011 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:31,555 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:31,968 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:34:31,970 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:32,306 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:34:32,306 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:34:32,547 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:33,012 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:33,547 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:34,547 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:35,543 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:35,888 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:34:35,889 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:35,889 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:35,950 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 21:34:35,951 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 21:34:36,013 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:34:36,013 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:34:36,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2024-12-31 21:34:36,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media +2024-12-31 21:34:36,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table/eval +2024-12-31 21:34:36,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table +2024-12-31 21:34:36,166 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:34:36,220 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/kgcipoja-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2024-12-31 21:34:36,407 INFO wandb-upload_2:1746604 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpbsw0m1e3 +2024-12-31 21:34:36,500 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 21:34:36,509 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:34:36,542 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:36,787 INFO SenderThread:1746604 [sender.py:send_request_log_artifact():1525] logged artifact run-z0y7vp39-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDEzMDA4OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE2ODUyMg==', 'latestArtifact': None}} +2024-12-31 21:34:36,787 DEBUG SenderThread:1746604 [sender.py:send():379] send: files +2024-12-31 21:34:36,788 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json with policy now +2024-12-31 21:34:36,788 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:34:36,788 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,788 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,788 DEBUG SenderThread:1746604 [sender.py:send():379] send: telemetry +2024-12-31 21:34:36,789 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,789 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,789 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,789 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,790 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,790 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,790 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,792 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,792 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,792 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,792 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,793 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,793 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,793 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,793 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,794 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,794 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,794 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,794 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,794 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,795 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:36,795 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:36,795 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 21:34:37,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:37,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:34:37,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:34:37,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json +2024-12-31 21:34:37,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table/eval +2024-12-31 21:34:37,027 INFO wandb-upload_1:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/2v5ebhzu-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2024-12-31 21:34:37,184 INFO wandb-upload_2:1746604 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpbptswpsg +2024-12-31 21:34:37,249 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/c6552hiw-media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json +2024-12-31 21:34:37,556 INFO SenderThread:1746604 [sender.py:send_request_log_artifact():1525] logged artifact run-z0y7vp39-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDEzMDA5Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE2ODUyMw==', 'latestArtifact': None}} +2024-12-31 21:34:37,556 DEBUG SenderThread:1746604 [sender.py:send():379] send: files +2024-12-31 21:34:37,556 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:37,556 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json with policy now +2024-12-31 21:34:37,556 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:34:37,557 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:37,557 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:37,557 DEBUG SenderThread:1746604 [sender.py:send():379] send: exit +2024-12-31 21:34:37,557 INFO SenderThread:1746604 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 21:34:37,558 INFO SenderThread:1746604 [sender.py:send_exit():588] handling runtime: 425 +2024-12-31 21:34:37,561 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:37,561 INFO SenderThread:1746604 [sender.py:send_exit():594] send defer +2024-12-31 21:34:37,561 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:37,561 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 21:34:37,562 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:37,562 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 21:34:37,562 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 1 +2024-12-31 21:34:37,562 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:37,562 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 21:34:37,562 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:37,562 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 21:34:37,562 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 2 +2024-12-31 21:34:37,562 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:37,562 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 21:34:37,562 INFO HandlerThread:1746604 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 21:34:37,562 DEBUG SystemMonitor:1746604 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 21:34:37,562 DEBUG SystemMonitor:1746604 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 21:34:37,564 INFO HandlerThread:1746604 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 21:34:37,564 INFO HandlerThread:1746604 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 21:34:37,635 INFO HandlerThread:1746604 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 21:34:37,635 INFO HandlerThread:1746604 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 21:34:37,635 INFO HandlerThread:1746604 [interfaces.py:finish():200] Joined network monitor +2024-12-31 21:34:37,635 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:37,635 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 21:34:37,635 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 3 +2024-12-31 21:34:37,636 DEBUG SenderThread:1746604 [sender.py:send():379] send: stats +2024-12-31 21:34:37,636 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:37,636 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 21:34:37,636 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:37,636 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 21:34:37,636 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 4 +2024-12-31 21:34:37,636 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:37,636 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 21:34:37,796 INFO wandb-upload_1:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/3jae89wk-media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json +2024-12-31 21:34:38,014 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:34:39,015 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:43,892 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:34:43,892 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:43,892 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 21:34:43,893 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:43,893 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 21:34:43,893 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 5 +2024-12-31 21:34:43,894 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:34:43,894 DEBUG SenderThread:1746604 [sender.py:send():379] send: history +2024-12-31 21:34:43,894 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:34:43,894 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:43,894 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 21:34:43,895 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:43,895 DEBUG SenderThread:1746604 [sender.py:send():379] send: summary +2024-12-31 21:34:43,898 INFO SenderThread:1746604 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:34:43,899 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:43,899 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 21:34:43,899 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 6 +2024-12-31 21:34:43,899 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:43,899 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 21:34:43,899 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:43,899 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 21:34:43,903 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:34:44,044 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 7 +2024-12-31 21:34:44,044 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:44,044 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 21:34:44,045 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:44,045 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 21:34:44,045 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:34:44,530 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:34:44,985 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 8 +2024-12-31 21:34:44,985 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:34:44,985 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:44,985 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 21:34:44,986 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:44,986 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 21:34:44,986 INFO SenderThread:1746604 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 21:34:44,988 INFO SenderThread:1746604 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 21:34:44,990 INFO SenderThread:1746604 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 21:34:45,004 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 9 +2024-12-31 21:34:45,005 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:45,005 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 21:34:45,005 DEBUG SenderThread:1746604 [sender.py:send():379] send: artifact +2024-12-31 21:34:45,045 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:45,046 INFO Thread-12 :1746604 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/config.yaml +2024-12-31 21:34:45,327 INFO wandb-upload_2:1746604 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmvus7pfh +2024-12-31 21:34:45,430 INFO wandb-upload_0:1746604 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpdf8zvwtj +2024-12-31 21:34:45,504 INFO wandb-upload_1:1746604 [upload_job.py:push():88] Uploaded file /tmp/tmpgdtnwb46/wandb-job.json +2024-12-31 21:34:45,530 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:34:46,003 INFO SenderThread:1746604 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDEzMDIwNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDEyMTU3MQ==', 'versionIndex': 392}}} +2024-12-31 21:34:46,003 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:46,003 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 21:34:46,003 INFO SenderThread:1746604 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 21:34:46,046 INFO SenderThread:1746604 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files +2024-12-31 21:34:46,046 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/config.yaml config.yaml +2024-12-31 21:34:46,046 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json wandb-summary.json +2024-12-31 21:34:46,047 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:34:46,047 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/requirements.txt requirements.txt +2024-12-31 21:34:46,047 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-metadata.json wandb-metadata.json +2024-12-31 21:34:46,047 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log output.log +2024-12-31 21:34:46,047 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/diff.patch diff.patch +2024-12-31 21:34:46,051 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 21:34:46,057 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/4_eval_4_4b071c579be3fd81d6bd.mp4 media/videos/eval/4_eval_4_4b071c579be3fd81d6bd.mp4 +2024-12-31 21:34:46,057 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/2_eval_2_8637db17a31d8f9e9a53.mp4 media/videos/eval/2_eval_2_8637db17a31d8f9e9a53.mp4 +2024-12-31 21:34:46,057 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/1_eval_1_eb820ba01fc256486a6c.mp4 media/videos/eval/1_eval_1_eb820ba01fc256486a6c.mp4 +2024-12-31 21:34:46,057 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/3_eval_3_4f1bea0839448d87829b.mp4 media/videos/eval/3_eval_3_4f1bea0839448d87829b.mp4 +2024-12-31 21:34:46,057 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/videos/eval/0_eval_0_73394dae2cb5c0e3e76d.mp4 media/videos/eval/0_eval_0_73394dae2cb5c0e3e76d.mp4 +2024-12-31 21:34:46,057 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2024-12-31 21:34:46,057 INFO SenderThread:1746604 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json +2024-12-31 21:34:46,057 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 10 +2024-12-31 21:34:46,058 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:34:46,058 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:46,058 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 21:34:46,058 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:46,058 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 21:34:46,058 INFO SenderThread:1746604 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:34:46,227 INFO wandb-upload_2:1746604 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/config.yaml +2024-12-31 21:34:46,231 INFO wandb-upload_0:1746604 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/wandb-summary.json +2024-12-31 21:34:46,284 INFO wandb-upload_3:1746604 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/requirements.txt +2024-12-31 21:34:46,293 INFO wandb-upload_4:1746604 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/files/output.log +2024-12-31 21:34:46,320 INFO wandb-upload_1:1746604 [upload_job.py:push():130] Uploaded file /tmp/tmpx5f7rtbvwandb/qtrx7o4i-events.out.tfevents.1735698452.leonmkim-ROG-Strix-G15CS-G15CS.1746382.0 +2024-12-31 21:34:46,521 INFO Thread-11 (_thread_body):1746604 [sender.py:transition_state():614] send defer: 11 +2024-12-31 21:34:46,521 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:46,521 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 21:34:46,521 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:46,521 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 21:34:46,521 INFO SenderThread:1746604 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 21:34:46,523 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 12 +2024-12-31 21:34:46,523 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:46,523 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 21:34:46,523 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:46,523 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 21:34:46,523 INFO SenderThread:1746604 [file_stream.py:finish():601] file stream finish called +2024-12-31 21:34:46,531 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:34:46,825 INFO SenderThread:1746604 [file_stream.py:finish():605] file stream finish is done +2024-12-31 21:34:46,825 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 13 +2024-12-31 21:34:46,825 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:34:46,826 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:46,826 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 21:34:46,826 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:46,826 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 21:34:46,826 INFO SenderThread:1746604 [sender.py:transition_state():614] send defer: 14 +2024-12-31 21:34:46,826 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:34:46,826 INFO HandlerThread:1746604 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 21:34:46,826 DEBUG SenderThread:1746604 [sender.py:send():379] send: final +2024-12-31 21:34:46,827 DEBUG SenderThread:1746604 [sender.py:send():379] send: footer +2024-12-31 21:34:46,827 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: defer +2024-12-31 21:34:46,827 INFO SenderThread:1746604 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 21:34:46,827 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:34:46,827 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:34:46,827 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 21:34:46,828 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:34:46,828 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 21:34:46,829 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 21:34:46,829 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: server_info +2024-12-31 21:34:46,831 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:34:46,884 DEBUG SenderThread:1746604 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:34:46,884 INFO MainThread:1746604 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 21:34:46,884 INFO MainThread:1746604 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 21:34:46,885 INFO MainThread:1746604 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 21:34:46,885 DEBUG HandlerThread:1746604 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 21:34:46,885 INFO HandlerThread:1746604 [handler.py:finish():882] shutting down handler +2024-12-31 21:34:47,829 INFO WriterThread:1746604 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/run-z0y7vp39.wandb +2024-12-31 21:34:47,884 INFO SenderThread:1746604 [sender.py:finish():1615] shutting down sender +2024-12-31 21:34:47,884 INFO SenderThread:1746604 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:34:47,884 INFO SenderThread:1746604 [file_pusher.py:join():175] waiting for file pusher diff --git a/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/logs/debug.log b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..920d32c02dfeb11e187639cd0e347df86a1d5d0d --- /dev/null +++ b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-31 21:27:31,564 INFO MainThread:1746382 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 21:27:31,565 INFO MainThread:1746382 [wandb_setup.py:_flush():76] Configure stats pid to 1746382 +2024-12-31 21:27:31,565 INFO MainThread:1746382 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 21:27:31,565 INFO MainThread:1746382 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/settings +2024-12-31 21:27:31,565 INFO MainThread:1746382 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 21:27:31,565 INFO MainThread:1746382 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 21:27:31,565 INFO MainThread:1746382 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 21:27:31,565 INFO MainThread:1746382 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 21:27:31,566 INFO MainThread:1746382 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/logs/debug.log +2024-12-31 21:27:31,566 INFO MainThread:1746382 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/logs/debug-internal.log +2024-12-31 21:27:31,566 INFO MainThread:1746382 [wandb_init.py:init():569] calling init triggers +2024-12-31 21:27:31,566 INFO MainThread:1746382 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39'} +2024-12-31 21:27:31,567 INFO MainThread:1746382 [wandb_init.py:init():619] starting backend +2024-12-31 21:27:31,567 INFO MainThread:1746382 [wandb_init.py:init():623] setting up manager +2024-12-31 21:27:31,570 INFO MainThread:1746382 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 21:27:31,572 INFO MainThread:1746382 [wandb_init.py:init():631] backend started and connected +2024-12-31 21:27:31,583 INFO MainThread:1746382 [wandb_init.py:init():720] updated telemetry +2024-12-31 21:27:31,590 INFO MainThread:1746382 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 21:27:31,869 INFO MainThread:1746382 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 21:27:31,906 INFO MainThread:1746382 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 21:27:31,907 INFO MainThread:1746382 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 21:27:32,283 INFO MainThread:1746382 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 21:27:32,283 INFO MainThread:1746382 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 21:27:32,283 INFO MainThread:1746382 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 21:27:32,283 INFO MainThread:1746382 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 21:27:32,285 INFO MainThread:1746382 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 21:27:32,285 INFO MainThread:1746382 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/z0y7vp39/tb, True +2024-12-31 21:28:27,244 INFO MainThread:1746382 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2024-12-31 21:34:47,885 WARNING MsgRouterThr:1746382 [router.py:message_loop():77] message_loop has been closed diff --git a/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/run-z0y7vp39.wandb b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/run-z0y7vp39.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a4fdbaa907c50b38ac8ec5f382e72626de3a61b9 --- /dev/null +++ b/z0y7vp39/wandb/run-20241231_212731-z0y7vp39/run-z0y7vp39.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fddf41972a16f99b91fc7f9de57bbfdcbf48e3ec26cd7ef420c60cf0a9275a99 +size 227775