diff --git a/.gitattributes b/.gitattributes index 9d92971654df953496b576d1b37ba496607ef9d7..e248dcf6c12a196f734e7abb7048b70b822eb65d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -108,3 +108,47 @@ c7zbixne/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text c7zbixne/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text c7zbixne/episode_rosbags/episode_1_2024-12-31-21-36-35.bag filter=lfs diff=lfs merge=lfs -text c7zbixne/episode_rosbags/episode_2_2024-12-31-21-37-20.bag filter=lfs diff=lfs merge=lfs -text +c7zbixne/episode_rosbags/episode_4_2024-12-31-21-39-09.bag filter=lfs diff=lfs merge=lfs -text +5r8mx4bu/episode_rosbags/episode_4_2025-01-01-00-04-57.bag filter=lfs diff=lfs merge=lfs -text +c7zbixne/episode_rosbags/episode_3_2024-12-31-21-38-07.bag filter=lfs diff=lfs merge=lfs -text +lydwde6h/episode_rosbags/episode_1_2025-01-01-11-17-32.bag filter=lfs diff=lfs merge=lfs -text +5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/1_eval_1_5acda3cc6f6a308de072.mp4 filter=lfs diff=lfs merge=lfs -text +5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/0_eval_0_bb2664eb51731886640b.mp4 filter=lfs diff=lfs merge=lfs -text +5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/4_eval_4_0f9f6705ea3bb2da7eeb.mp4 filter=lfs diff=lfs merge=lfs -text +5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/2_eval_2_ae958bbddc0d8e506e3a.mp4 filter=lfs diff=lfs merge=lfs -text +5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/3_eval_3_ca451a47ae860c604497.mp4 filter=lfs diff=lfs merge=lfs -text +5r8mx4bu/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5r8mx4bu/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5r8mx4bu/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5r8mx4bu/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5r8mx4bu/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +c7zbixne/episode_rosbags/episode_0_2024-12-31-21-35-34.bag filter=lfs diff=lfs merge=lfs -text +5r8mx4bu/episode_rosbags/episode_3_2025-01-01-00-03-49.bag filter=lfs diff=lfs merge=lfs -text +5r8mx4bu/episode_rosbags/episode_1_2025-01-01-00-01-46.bag filter=lfs diff=lfs merge=lfs -text +hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/1_eval_1_fae1d5a4084c3fa7e4ce.mp4 filter=lfs diff=lfs merge=lfs -text +iz44h5u3/episode_rosbags/episode_2_2025-01-01-11-54-28.bag filter=lfs diff=lfs merge=lfs -text +iz44h5u3/episode_rosbags/episode_4_2025-01-01-11-56-13.bag filter=lfs diff=lfs merge=lfs -text +hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/4_eval_4_20669f7dea9bec0e6b76.mp4 filter=lfs diff=lfs merge=lfs -text +hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/3_eval_3_26ea14ca5e0897c83196.mp4 filter=lfs diff=lfs merge=lfs -text +hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/2_eval_2_07f9274f756bd81cbaee.mp4 filter=lfs diff=lfs merge=lfs -text +hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/0_eval_0_0a25a53b5c8820b669aa.mp4 filter=lfs diff=lfs merge=lfs -text +hex7gk4r/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hex7gk4r/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hex7gk4r/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hex7gk4r/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hex7gk4r/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +iz44h5u3/episode_rosbags/episode_1_2025-01-01-11-53-32.bag filter=lfs diff=lfs merge=lfs -text +iz44h5u3/episode_rosbags/episode_3_2025-01-01-11-55-18.bag filter=lfs diff=lfs merge=lfs -text +7j9cp4xk/episode_rosbags/episode_0_2025-01-01-00-07-19.bag filter=lfs diff=lfs merge=lfs -text +5r8mx4bu/episode_rosbags/episode_2_2025-01-01-00-02-45.bag filter=lfs diff=lfs merge=lfs -text +8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/1_eval_1_d616a9f6d337515bde74.mp4 filter=lfs diff=lfs merge=lfs -text +8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/2_eval_2_52a5d239f2ed3d88df0e.mp4 filter=lfs diff=lfs merge=lfs -text +8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/3_eval_3_c6b478b7570f0157dada.mp4 filter=lfs diff=lfs merge=lfs -text +8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/4_eval_4_b9e904a3822ce75d6da0.mp4 filter=lfs diff=lfs merge=lfs -text +8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/0_eval_0_080cfd687bc0e6205960.mp4 filter=lfs diff=lfs merge=lfs -text +8qih81be/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8qih81be/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8qih81be/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8qih81be/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8qih81be/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +lydwde6h/episode_rosbags/episode_0_2025-01-01-11-16-24.bag filter=lfs diff=lfs merge=lfs -text diff --git a/5r8mx4bu/.hydra/config.yaml b/5r8mx4bu/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05485f98b3c08bece875a460fe25a0cce7504f90 --- /dev/null +++ b/5r8mx4bu/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1006_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/5r8mx4bu/.hydra/hydra.yaml b/5r8mx4bu/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..360fb305f87ab4eea850be32a4ce558c797b1a31 --- /dev/null +++ b/5r8mx4bu/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/5r8mx4bu/.hydra/overrides.yaml b/5r8mx4bu/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/5r8mx4bu/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/5r8mx4bu/episode_rosbags/color_K.npy b/5r8mx4bu/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/5r8mx4bu/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/5r8mx4bu/episode_rosbags/episode_1_2025-01-01-00-01-46.bag b/5r8mx4bu/episode_rosbags/episode_1_2025-01-01-00-01-46.bag new file mode 100644 index 0000000000000000000000000000000000000000..fba877582cb14ae2fd456dd9e6db6a42fd23c7e8 --- /dev/null +++ b/5r8mx4bu/episode_rosbags/episode_1_2025-01-01-00-01-46.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cba59bd77954812c41eaabead298882928d96247c7fc422fadb4f9dba2615326 +size 2037391671 diff --git a/5r8mx4bu/episode_rosbags/episode_2_2025-01-01-00-02-45.bag b/5r8mx4bu/episode_rosbags/episode_2_2025-01-01-00-02-45.bag new file mode 100644 index 0000000000000000000000000000000000000000..377d7552a8d327405dc58cd34fc8deb0ef19514f --- /dev/null +++ b/5r8mx4bu/episode_rosbags/episode_2_2025-01-01-00-02-45.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c955f517428140665772982034d08688da6cb091516ece18d412241be339363d +size 1539311667 diff --git a/5r8mx4bu/episode_rosbags/episode_3_2025-01-01-00-03-49.bag b/5r8mx4bu/episode_rosbags/episode_3_2025-01-01-00-03-49.bag new file mode 100644 index 0000000000000000000000000000000000000000..a30e787ba4dae42a8952cc37ef313a64adcc705e --- /dev/null +++ b/5r8mx4bu/episode_rosbags/episode_3_2025-01-01-00-03-49.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d22f124af30df1391601f94f14e46c2fc83a4d2526c98825309104facd0e16c +size 2032615376 diff --git a/5r8mx4bu/episode_rosbags/episode_4_2025-01-01-00-04-57.bag b/5r8mx4bu/episode_rosbags/episode_4_2025-01-01-00-04-57.bag new file mode 100644 index 0000000000000000000000000000000000000000..984985522e803c38068ec0fa2e57c9a0a8ea84c3 --- /dev/null +++ b/5r8mx4bu/episode_rosbags/episode_4_2025-01-01-00-04-57.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6000c375c85a27795e8e770ef5c1a52a98b1870ca55e593a96c6d5818c65c9b4 +size 1206972646 diff --git a/5r8mx4bu/eval_video/0_eval.mp4 b/5r8mx4bu/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7a5f8802b06c6b4ec8f8d40b9e085c4be208491b --- /dev/null +++ b/5r8mx4bu/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb2664eb51731886640bdaa2f28ef878fb994a6fb7b50cd6e63c1396b1496bc4 +size 1470956 diff --git a/5r8mx4bu/eval_video/1_eval.mp4 b/5r8mx4bu/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df0a06a61d16765bfa2bccb4ec05d4f8a7947a4a --- /dev/null +++ b/5r8mx4bu/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5acda3cc6f6a308de072312035849b844cad8f518137c51f1d20363772ff66a0 +size 1191997 diff --git a/5r8mx4bu/eval_video/2_eval.mp4 b/5r8mx4bu/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..89fbb6b8dddad7781185c730c9db71a17bbd7d01 --- /dev/null +++ b/5r8mx4bu/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae958bbddc0d8e506e3adfb357cd37bb6d3cc9284e0b259a0cd78c092d04b294 +size 930968 diff --git a/5r8mx4bu/eval_video/3_eval.mp4 b/5r8mx4bu/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4bcd5c16b64542191693ce55c3b0d99363e9a3aa --- /dev/null +++ b/5r8mx4bu/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca451a47ae860c60449746fb81d545cf8171ef7ecde64b9b7d9b3b0a8544de94 +size 1183575 diff --git a/5r8mx4bu/eval_video/4_eval.mp4 b/5r8mx4bu/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a430dfac9d86361168f1c84cb15f50573ab65b76 --- /dev/null +++ b/5r8mx4bu/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f9f6705ea3bb2da7eebe4e623b267c259b890d89f31660bf2dda08db8e8dbf3 +size 753562 diff --git a/5r8mx4bu/tb/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 b/5r8mx4bu/tb/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 new file mode 100644 index 0000000000000000000000000000000000000000..7368e33a30883b7178c57ecdc03657af5af1ca27 --- /dev/null +++ b/5r8mx4bu/tb/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f113f05571094dbe8fbb9295687ea5bb8e6a81d5bc95760f1a8823cb0a350bb +size 6539 diff --git a/5r8mx4bu/wandb/debug-internal.log b/5r8mx4bu/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5cdb244818596d6e6f712691f2295df29314a2fd --- /dev/null +++ b/5r8mx4bu/wandb/debug-internal.log @@ -0,0 +1,902 @@ +2024-12-31 23:59:53,328 INFO StreamThr :1772506 [internal.py:wandb_internal():85] W&B internal server running at pid: 1772506, started at: 2024-12-31 23:59:53.328438 +2024-12-31 23:59:53,330 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status +2024-12-31 23:59:53,342 INFO WriterThread:1772506 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/run-5r8mx4bu.wandb +2024-12-31 23:59:53,342 DEBUG SenderThread:1772506 [sender.py:send():379] send: header +2024-12-31 23:59:53,354 DEBUG SenderThread:1772506 [sender.py:send():379] send: run +2024-12-31 23:59:53,633 INFO SenderThread:1772506 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files +2024-12-31 23:59:53,633 INFO SenderThread:1772506 [sender.py:_start_run_threads():1188] run started: 5r8mx4bu with start time 1735707593.328535 +2024-12-31 23:59:53,651 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 23:59:53,652 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: check_version +2024-12-31 23:59:53,737 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 23:59:53,745 DEBUG HandlerThread:1772506 [system_info.py:__init__():26] System info init +2024-12-31 23:59:53,745 DEBUG HandlerThread:1772506 [system_info.py:__init__():41] System info init done +2024-12-31 23:59:53,745 INFO HandlerThread:1772506 [system_monitor.py:start():194] Starting system monitor +2024-12-31 23:59:53,745 INFO SystemMonitor:1772506 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 23:59:53,745 INFO HandlerThread:1772506 [system_monitor.py:probe():214] Collecting system info +2024-12-31 23:59:53,746 INFO SystemMonitor:1772506 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 23:59:53,747 INFO SystemMonitor:1772506 [interfaces.py:start():188] Started disk monitoring +2024-12-31 23:59:53,748 INFO SystemMonitor:1772506 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 23:59:53,748 INFO SystemMonitor:1772506 [interfaces.py:start():188] Started memory monitoring +2024-12-31 23:59:53,749 INFO SystemMonitor:1772506 [interfaces.py:start():188] Started network monitoring +2024-12-31 23:59:53,754 DEBUG HandlerThread:1772506 [system_info.py:probe():152] Probing system +2024-12-31 23:59:53,756 DEBUG HandlerThread:1772506 [system_info.py:_probe_git():137] Probing git +2024-12-31 23:59:53,761 DEBUG HandlerThread:1772506 [system_info.py:_probe_git():145] Probing git done +2024-12-31 23:59:53,761 DEBUG HandlerThread:1772506 [system_info.py:probe():200] Probing system done +2024-12-31 23:59:53,761 DEBUG HandlerThread:1772506 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T04:59:53.754835', 'startedAt': '2025-01-01T04:59:53.320075', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.501, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.987, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.045, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 632.6411437988281}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 23:59:53,761 INFO HandlerThread:1772506 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 23:59:53,761 INFO HandlerThread:1772506 [system_monitor.py:probe():227] Publishing system info +2024-12-31 23:59:53,761 DEBUG HandlerThread:1772506 [system_info.py:_save_code():44] Saving code +2024-12-31 23:59:53,763 DEBUG HandlerThread:1772506 [system_info.py:_save_code():67] Saving code done +2024-12-31 23:59:53,763 DEBUG HandlerThread:1772506 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 23:59:53,824 DEBUG HandlerThread:1772506 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 23:59:53,824 INFO HandlerThread:1772506 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 23:59:53,828 DEBUG SenderThread:1772506 [sender.py:send():379] send: files +2024-12-31 23:59:53,828 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 23:59:53,828 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 23:59:53,828 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 23:59:54,061 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 23:59:54,062 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: python_packages +2024-12-31 23:59:54,062 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/z3trdkb9-wandb-metadata.json +2024-12-31 23:59:54,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:59:54,063 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:59:54,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:59:54,065 INFO HandlerThread:1772506 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/tb" + save: true +} +control { + relay_id: "b88240e5503a46e9a3ae165fd55e2f33" +} +_info { + stream_id: "5r8mx4bu" +} + +2024-12-31 23:59:54,065 DEBUG HandlerThread:1772506 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 23:59:54,106 INFO wandb-upload_2:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/39i0rf51-diff.patch +2024-12-31 23:59:54,145 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/m6vzfnnh-code/FISH/eval_robot.py +2024-12-31 23:59:54,170 DEBUG SenderThread:1772506 [sender.py:send():379] send: telemetry +2024-12-31 23:59:54,170 DEBUG SenderThread:1772506 [sender.py:send():379] send: tbrecord +2024-12-31 23:59:54,171 DEBUG SenderThread:1772506 [sender.py:send():379] send: files +2024-12-31 23:59:54,171 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 with policy live +2024-12-31 23:59:54,368 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/4p2z9ne0-events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2024-12-31 23:59:54,635 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/code/FISH/eval_robot.py +2024-12-31 23:59:54,635 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/requirements.txt +2024-12-31 23:59:54,635 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2024-12-31 23:59:54,635 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2024-12-31 23:59:54,635 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/diff.patch +2024-12-31 23:59:54,636 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-metadata.json +2024-12-31 23:59:54,636 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/code/FISH +2024-12-31 23:59:54,636 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/code +2024-12-31 23:59:55,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:59:56,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:59:56,634 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2024-12-31 23:59:57,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:59:58,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:59:58,426 DEBUG SenderThread:1772506 [sender.py:send():379] send: config +2024-12-31 23:59:58,426 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:59:59,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:00,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:00,635 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:01,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:02,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:03,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:03,434 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:04,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:04,637 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:05,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:06,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:07,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:08,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:08,536 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:09,062 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:00:09,062 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:00:09,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:10,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:10,639 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:11,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:12,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:13,065 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:14,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:14,065 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:14,641 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:15,065 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:16,065 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:17,065 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:18,065 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:18,642 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:19,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:19,066 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:20,066 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:20,642 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:21,066 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:22,066 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:22,643 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:23,066 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:24,062 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:00:24,062 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:00:24,105 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:24,200 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:24,643 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/config.yaml +2025-01-01 00:00:25,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:26,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:27,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:28,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:29,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:30,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:30,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:31,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:32,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:32,646 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:33,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:34,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:35,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:35,068 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:36,068 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:37,068 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:38,068 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:38,647 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:39,062 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:00:39,062 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:00:39,105 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:40,066 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:40,068 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:41,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:42,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:42,648 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:43,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:44,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:45,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:46,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:46,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:47,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:48,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:49,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:50,071 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:51,071 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:51,225 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:52,071 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:52,651 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:53,071 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:53,749 DEBUG SystemMonitor:1772506 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 00:00:53,750 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:00:54,062 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:00:54,062 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:00:54,105 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:55,071 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:56,071 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:57,071 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:57,146 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:58,072 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:59,072 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:00,072 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:01,072 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:02,072 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:03,065 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:03,072 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:04,072 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:05,072 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:06,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:07,072 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:08,065 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:08,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:09,062 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:01:09,062 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:01:09,109 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:10,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:11,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:12,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:13,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:13,198 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:14,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:15,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:16,062 DEBUG SenderThread:1772506 [sender.py:send():379] send: files +2025-01-01 00:01:16,062 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_bb2664eb51731886640b.mp4 with policy now +2025-01-01 00:01:16,062 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:01:16,063 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:01:16,063 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:01:16,065 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:01:16,075 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:16,330 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/0bwxxs12-media/videos/eval/0_eval_0_bb2664eb51731886640b.mp4 +2025-01-01 00:01:16,658 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:16,658 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:01:16,659 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/0_eval_0_bb2664eb51731886640b.mp4 +2025-01-01 00:01:16,659 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media +2025-01-01 00:01:16,659 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval +2025-01-01 00:01:16,659 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos +2025-01-01 00:01:16,933 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/0r8fzcd1-media/videos/eval/0_eval_0_bb2664eb51731886640b.mp4 +2025-01-01 00:01:17,074 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:18,076 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:18,659 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:19,065 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:19,075 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:20,074 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:21,113 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:22,078 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:22,660 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:23,078 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:23,750 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:01:24,062 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:01:24,062 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:01:24,105 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:24,173 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:25,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:26,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:27,078 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:28,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:28,662 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:29,078 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:29,887 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:30,078 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:31,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:32,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:32,663 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:33,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:33,664 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:01:33,930 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/bt73oq8c-events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:01:34,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:34,664 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:35,066 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:35,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:36,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:37,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:38,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:39,062 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:01:39,063 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:01:39,109 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:40,080 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:40,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:40,666 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:41,080 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:42,080 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:43,080 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:44,080 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:44,668 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:45,080 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:45,169 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:46,080 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:46,668 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:47,081 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:48,081 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:48,669 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:49,081 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:50,081 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:51,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:51,082 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:52,081 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:53,081 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:53,751 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:01:54,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:01:54,063 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:01:54,105 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:55,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:56,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:56,669 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:56,672 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:57,082 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:58,082 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:59,082 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:00,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:01,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:02,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:02,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:03,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:04,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:04,674 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:05,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:06,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:07,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:07,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:08,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:09,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:02:09,063 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:02:09,105 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:10,084 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:11,084 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:12,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:12,084 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:13,084 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:14,084 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:14,677 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:15,084 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:15,678 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/1_eval_1_5acda3cc6f6a308de072.mp4 +2025-01-01 00:02:15,678 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval +2025-01-01 00:02:15,684 DEBUG SenderThread:1772506 [sender.py:send():379] send: files +2025-01-01 00:02:15,684 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_5acda3cc6f6a308de072.mp4 with policy now +2025-01-01 00:02:15,685 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:02:15,686 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:02:15,686 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:02:15,686 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:02:15,941 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/go50pzf2-media/videos/eval/1_eval_1_5acda3cc6f6a308de072.mp4 +2025-01-01 00:02:16,085 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:16,209 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/zxu8ovf9-media/videos/eval/1_eval_1_5acda3cc6f6a308de072.mp4 +2025-01-01 00:02:16,678 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:16,678 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:02:17,087 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:17,678 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:17,714 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:18,090 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:19,085 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:20,093 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:21,094 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:22,091 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:23,075 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:23,093 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:23,751 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:02:24,068 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:02:24,068 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:02:24,113 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:25,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:26,097 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:26,681 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:27,101 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:28,098 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:28,165 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:28,682 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:29,095 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:30,095 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:30,683 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:31,095 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:32,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:33,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:34,069 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:34,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:35,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:36,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:37,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:38,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:39,066 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:02:39,066 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:02:39,109 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:39,181 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:40,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:41,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:42,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:42,631 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:02:42,632 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:02:42,632 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:02:42,686 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:02:42,687 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:02:42,938 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/ptwszs2x-events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:02:43,097 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:44,097 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:44,687 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:45,069 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:45,097 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:46,098 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:46,688 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:47,098 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:48,098 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:48,689 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:49,099 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:50,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:50,101 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:51,100 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:52,102 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:53,102 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:53,752 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:02:54,066 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:02:54,066 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:02:54,109 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:55,102 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:55,188 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:56,102 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:57,102 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:58,103 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:59,103 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:00,103 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:01,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:01,103 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:02,103 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:03,103 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:04,104 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:05,104 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:06,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:06,104 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:07,105 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:07,713 DEBUG SenderThread:1772506 [sender.py:send():379] send: files +2025-01-01 00:03:07,713 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_ae958bbddc0d8e506e3a.mp4 with policy now +2025-01-01 00:03:07,716 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:03:07,716 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:03:07,716 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:03:07,717 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:03:07,965 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/81v92fm9-media/videos/eval/2_eval_2_ae958bbddc0d8e506e3a.mp4 +2025-01-01 00:03:08,104 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:08,695 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:08,696 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:03:08,696 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/2_eval_2_ae958bbddc0d8e506e3a.mp4 +2025-01-01 00:03:08,696 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval +2025-01-01 00:03:09,006 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/qy3vxjrr-media/videos/eval/2_eval_2_ae958bbddc0d8e506e3a.mp4 +2025-01-01 00:03:09,081 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:03:09,081 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:03:09,125 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:10,109 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:11,113 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:11,226 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:12,114 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:13,111 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:14,120 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:14,698 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:15,116 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:16,119 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:17,082 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:17,117 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:18,126 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:18,699 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:19,121 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:20,130 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:20,700 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:21,123 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:22,122 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:22,685 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:22,700 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:23,125 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:23,752 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:03:24,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:03:24,073 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:03:24,127 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:25,123 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:26,125 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:27,124 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:28,127 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:28,210 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:29,131 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:30,131 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:31,128 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:32,131 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:33,131 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:34,081 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:34,130 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:35,130 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:36,130 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:36,399 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:03:36,399 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:03:36,399 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:03:36,705 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:36,705 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:03:36,705 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:03:37,132 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:38,132 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:39,074 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:03:39,074 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:03:39,132 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:39,231 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:40,136 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:41,133 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:42,138 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:42,707 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:43,142 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:44,139 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:45,084 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:45,139 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:46,138 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:47,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:48,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:48,709 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:49,146 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:50,140 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:50,710 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:50,950 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:51,140 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:52,140 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:52,711 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:53,140 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:53,752 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:03:54,069 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:03:54,070 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:03:54,141 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:55,141 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:56,129 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:56,141 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:57,141 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:58,141 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:59,142 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:00,143 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:01,142 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:02,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:02,142 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:03,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:04,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:05,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:06,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:07,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:07,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:08,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:09,071 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:04:09,071 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:04:09,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:10,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:11,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:12,145 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:12,192 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:13,145 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:14,145 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:15,145 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:16,145 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:17,145 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:17,746 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:18,153 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:18,501 DEBUG SenderThread:1772506 [sender.py:send():379] send: files +2025-01-01 00:04:18,502 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ca451a47ae860c604497.mp4 with policy now +2025-01-01 00:04:18,505 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:04:18,506 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:04:18,506 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:04:18,506 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:04:18,720 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:18,720 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:04:18,720 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/3_eval_3_ca451a47ae860c604497.mp4 +2025-01-01 00:04:18,720 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval +2025-01-01 00:04:18,786 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/vjpaulnw-media/videos/eval/3_eval_3_ca451a47ae860c604497.mp4 +2025-01-01 00:04:19,075 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/4r8q538n-media/videos/eval/3_eval_3_ca451a47ae860c604497.mp4 +2025-01-01 00:04:19,156 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:19,720 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:20,156 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:20,721 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:21,156 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:22,151 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:23,087 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:23,155 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:23,753 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:04:24,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:04:24,073 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:04:24,162 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:24,722 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:25,162 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:26,158 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:27,158 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:28,162 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:28,187 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:29,162 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:30,165 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:30,724 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:31,170 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:32,171 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:33,168 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:33,469 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:34,167 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:34,725 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:35,171 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:36,167 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:37,173 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:38,172 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:39,077 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:04:39,078 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:04:39,173 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:39,223 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:40,172 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:41,172 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:42,170 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:43,173 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:43,728 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:04:43,997 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/05clmi3g-events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:04:44,172 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:44,482 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:04:44,483 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:04:44,483 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:44,484 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:04:44,729 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:44,729 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:04:44,729 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:04:45,176 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:46,178 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:47,176 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:48,180 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:49,178 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:50,090 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:50,186 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:50,731 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:51,178 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:52,184 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:53,181 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:53,753 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:04:54,075 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:04:54,075 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:04:54,179 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:54,732 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:55,183 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:55,937 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:56,184 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:56,733 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:57,183 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:58,182 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:58,733 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:59,182 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:00,182 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:01,021 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:01,182 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:02,182 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:02,735 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:03,182 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:04,182 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:05,183 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:06,089 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:06,183 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:07,183 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:08,183 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:09,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:05:09,073 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:05:09,183 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:10,183 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:11,183 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:11,195 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:12,184 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:13,184 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:14,221 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:15,194 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:15,273 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:05:15,273 DEBUG SenderThread:1772506 [sender.py:send():379] send: files +2025-01-01 00:05:15,273 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_0f9f6705ea3bb2da7eeb.mp4 with policy now +2025-01-01 00:05:15,274 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:05:15,274 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:15,277 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:15,512 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/06qmqt7h-media/videos/eval/4_eval_4_0f9f6705ea3bb2da7eeb.mp4 +2025-01-01 00:05:15,739 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:05:15,740 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/4_eval_4_0f9f6705ea3bb2da7eeb.mp4 +2025-01-01 00:05:15,740 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval +2025-01-01 00:05:15,991 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/y8po8ayg-media/videos/eval/4_eval_4_0f9f6705ea3bb2da7eeb.mp4 +2025-01-01 00:05:16,204 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:16,304 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:16,740 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:17,203 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:18,194 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:19,198 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:20,203 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:21,197 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:21,345 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:22,198 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:22,742 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:23,200 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:23,754 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:05:24,087 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:05:24,087 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:05:24,221 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:25,203 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:26,206 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:26,410 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:26,743 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:27,208 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:28,214 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:28,744 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:29,216 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:30,213 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:30,744 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:31,218 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:32,214 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:32,261 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:33,211 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:34,225 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:35,215 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:36,219 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:37,216 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:37,865 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:38,221 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:38,298 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 00:05:38,298 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 00:05:38,483 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:05:38,747 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:38,747 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:05:38,748 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 00:05:38,748 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media +2025-01-01 00:05:38,748 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table +2025-01-01 00:05:38,748 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table/eval +2025-01-01 00:05:38,785 INFO wandb-upload_0:1772506 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmdq_ekj4 +2025-01-01 00:05:38,886 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 00:05:38,895 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:05:38,938 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/5038he12-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 00:05:39,080 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:05:39,213 INFO SenderThread:1772506 [sender.py:send_request_log_artifact():1525] logged artifact run-5r8mx4bu-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjgzNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE4MDQ1MA==', 'latestArtifact': None}} +2025-01-01 00:05:39,213 DEBUG SenderThread:1772506 [sender.py:send():379] send: files +2025-01-01 00:05:39,213 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-01 00:05:39,213 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:05:39,213 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,214 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,214 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:05:39,214 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,214 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,215 DEBUG SenderThread:1772506 [sender.py:send():379] send: telemetry +2025-01-01 00:05:39,215 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,215 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,215 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,216 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,216 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:39,216 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,218 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,218 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,218 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,219 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,219 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,219 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,219 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,219 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,220 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,220 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,220 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,220 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,221 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,221 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,221 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,221 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,222 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,222 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 00:05:39,457 INFO wandb-upload_2:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/uqpqat07-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 00:05:39,665 INFO wandb-upload_1:1772506 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpq34rapwq +2025-01-01 00:05:39,749 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:05:39,749 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:05:39,749 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table/eval/total_success_rate_ci_6_b6adbd01d0fdba7c5d33.table.json +2025-01-01 00:05:39,749 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table/eval +2025-01-01 00:05:40,096 INFO SenderThread:1772506 [sender.py:send_request_log_artifact():1525] logged artifact run-5r8mx4bu-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjg0NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE4MDQ1MQ==', 'latestArtifact': None}} +2025-01-01 00:05:40,096 DEBUG SenderThread:1772506 [sender.py:send():379] send: files +2025-01-01 00:05:40,096 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_b6adbd01d0fdba7c5d33.table.json with policy now +2025-01-01 00:05:40,097 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:05:40,097 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:40,097 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:40,098 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:05:40,215 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:40,276 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/22khlc6s-media/table/eval/total_success_rate_ci_6_b6adbd01d0fdba7c5d33.table.json +2025-01-01 00:05:40,547 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/2kg0vga6-media/table/eval/total_success_rate_ci_6_b6adbd01d0fdba7c5d33.table.json +2025-01-01 00:05:40,749 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:40,750 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:05:41,215 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:42,209 DEBUG SenderThread:1772506 [sender.py:send():379] send: exit +2025-01-01 00:05:42,209 INFO SenderThread:1772506 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 00:05:42,210 INFO SenderThread:1772506 [sender.py:send_exit():588] handling runtime: 348 +2025-01-01 00:05:42,210 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:42,210 INFO SenderThread:1772506 [sender.py:send_exit():594] send defer +2025-01-01 00:05:42,210 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:42,210 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 00:05:42,210 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:42,210 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 00:05:42,210 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 1 +2025-01-01 00:05:42,211 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:42,211 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 00:05:42,211 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:42,211 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 00:05:42,211 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 2 +2025-01-01 00:05:42,211 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:42,211 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 00:05:42,211 INFO HandlerThread:1772506 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 00:05:42,211 DEBUG SystemMonitor:1772506 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 00:05:42,211 DEBUG SystemMonitor:1772506 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 00:05:42,212 INFO HandlerThread:1772506 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 00:05:42,212 INFO HandlerThread:1772506 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 00:05:42,262 INFO HandlerThread:1772506 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 00:05:42,263 INFO HandlerThread:1772506 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 00:05:42,263 INFO HandlerThread:1772506 [interfaces.py:finish():200] Joined network monitor +2025-01-01 00:05:42,263 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:42,263 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 00:05:42,263 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 3 +2025-01-01 00:05:42,263 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:05:42,264 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:42,264 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 00:05:42,264 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:42,264 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 00:05:42,264 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 4 +2025-01-01 00:05:42,264 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:42,264 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 00:05:42,750 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:05:44,751 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:48,549 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:05:48,549 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:48,549 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 00:05:48,549 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:48,550 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:48,550 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 00:05:48,550 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 5 +2025-01-01 00:05:48,551 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:05:48,551 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:05:48,551 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:48,551 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:48,551 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 00:05:48,552 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:48,553 DEBUG SenderThread:1772506 [sender.py:send():379] send: summary +2025-01-01 00:05:48,553 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:48,554 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:48,554 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 00:05:48,554 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 6 +2025-01-01 00:05:48,554 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:48,554 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 00:05:48,554 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:48,554 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 00:05:48,554 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 7 +2025-01-01 00:05:48,554 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:48,554 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:48,554 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 00:05:48,555 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:48,555 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 00:05:48,753 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/config.yaml +2025-01-01 00:05:48,753 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:05:49,211 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:05:51,645 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 8 +2025-01-01 00:05:51,645 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:05:51,645 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:51,646 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 00:05:51,646 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:51,646 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 00:05:51,646 INFO SenderThread:1772506 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 00:05:51,648 INFO SenderThread:1772506 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 00:05:51,650 INFO SenderThread:1772506 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 00:05:51,663 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 9 +2025-01-01 00:05:51,664 DEBUG SenderThread:1772506 [sender.py:send():379] send: artifact +2025-01-01 00:05:51,667 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:51,667 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 00:05:51,754 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:52,033 INFO wandb-upload_1:1772506 [upload_job.py:push():85] Skipped uploading /tmp/tmptzi91ge5/wandb-job.json +2025-01-01 00:05:52,033 INFO wandb-upload_2:1772506 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9f8_6z8e +2025-01-01 00:05:52,154 INFO wandb-upload_0:1772506 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmps8pm0g3d +2025-01-01 00:05:52,211 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:05:52,687 INFO SenderThread:1772506 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjk2Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDIxODYyMQ==', 'versionIndex': 402}}} +2025-01-01 00:05:52,687 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:52,687 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 00:05:52,687 INFO SenderThread:1772506 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 00:05:52,755 INFO SenderThread:1772506 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files +2025-01-01 00:05:52,755 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/config.yaml config.yaml +2025-01-01 00:05:52,755 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json wandb-summary.json +2025-01-01 00:05:52,757 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/requirements.txt requirements.txt +2025-01-01 00:05:52,759 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-metadata.json wandb-metadata.json +2025-01-01 00:05:52,759 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:05:52,759 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log output.log +2025-01-01 00:05:52,760 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/diff.patch diff.patch +2025-01-01 00:05:52,761 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 00:05:52,761 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/3_eval_3_ca451a47ae860c604497.mp4 media/videos/eval/3_eval_3_ca451a47ae860c604497.mp4 +2025-01-01 00:05:52,761 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/2_eval_2_ae958bbddc0d8e506e3a.mp4 media/videos/eval/2_eval_2_ae958bbddc0d8e506e3a.mp4 +2025-01-01 00:05:52,761 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/4_eval_4_0f9f6705ea3bb2da7eeb.mp4 media/videos/eval/4_eval_4_0f9f6705ea3bb2da7eeb.mp4 +2025-01-01 00:05:52,761 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/0_eval_0_bb2664eb51731886640b.mp4 media/videos/eval/0_eval_0_bb2664eb51731886640b.mp4 +2025-01-01 00:05:52,761 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/1_eval_1_5acda3cc6f6a308de072.mp4 media/videos/eval/1_eval_1_5acda3cc6f6a308de072.mp4 +2025-01-01 00:05:52,761 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table/eval/total_success_rate_ci_6_b6adbd01d0fdba7c5d33.table.json media/table/eval/total_success_rate_ci_6_b6adbd01d0fdba7c5d33.table.json +2025-01-01 00:05:52,763 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 00:05:52,763 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 10 +2025-01-01 00:05:52,763 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:05:52,763 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:52,763 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 00:05:52,764 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:52,765 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 00:05:52,765 INFO SenderThread:1772506 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 00:05:52,956 INFO wandb-upload_2:1772506 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/requirements.txt +2025-01-01 00:05:52,977 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:05:52,980 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/config.yaml +2025-01-01 00:05:53,018 INFO wandb-upload_4:1772506 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:53,056 INFO wandb-upload_3:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/k4qf7v3f-events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:05:53,212 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:05:53,212 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:05:53,257 INFO Thread-11 (_thread_body):1772506 [sender.py:transition_state():614] send defer: 11 +2025-01-01 00:05:53,257 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:53,257 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 00:05:53,257 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:53,257 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 00:05:53,258 INFO SenderThread:1772506 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 00:05:53,259 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 12 +2025-01-01 00:05:53,259 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:53,259 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 00:05:53,259 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:53,259 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 00:05:53,259 INFO SenderThread:1772506 [file_stream.py:finish():601] file stream finish called +2025-01-01 00:05:53,423 INFO SenderThread:1772506 [file_stream.py:finish():605] file stream finish is done +2025-01-01 00:05:53,424 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 13 +2025-01-01 00:05:53,424 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:53,424 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 00:05:53,424 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:53,424 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 00:05:53,424 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 14 +2025-01-01 00:05:53,424 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:53,424 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 00:05:53,425 DEBUG SenderThread:1772506 [sender.py:send():379] send: final +2025-01-01 00:05:53,425 DEBUG SenderThread:1772506 [sender.py:send():379] send: footer +2025-01-01 00:05:53,425 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:53,425 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 00:05:53,425 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:05:53,426 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:05:53,426 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 00:05:53,426 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:05:53,427 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 00:05:53,428 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 00:05:53,429 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:53,429 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: server_info +2025-01-01 00:05:53,482 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:05:53,482 INFO MainThread:1772506 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 00:05:53,483 INFO MainThread:1772506 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 00:05:53,483 INFO MainThread:1772506 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 00:05:53,483 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 00:05:53,483 INFO HandlerThread:1772506 [handler.py:finish():882] shutting down handler +2025-01-01 00:05:54,427 INFO WriterThread:1772506 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/run-5r8mx4bu.wandb +2025-01-01 00:05:54,482 INFO SenderThread:1772506 [sender.py:finish():1615] shutting down sender +2025-01-01 00:05:54,482 INFO SenderThread:1772506 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 00:05:54,482 INFO SenderThread:1772506 [file_pusher.py:join():175] waiting for file pusher diff --git a/5r8mx4bu/wandb/debug.log b/5r8mx4bu/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..960da7ce724403a5583b366b95c424bed7459cd9 --- /dev/null +++ b/5r8mx4bu/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_setup.py:_flush():76] Configure stats pid to 1772311 +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/settings +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/logs/debug.log +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/logs/debug-internal.log +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_init.py:init():569] calling init triggers +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu'} +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_init.py:init():619] starting backend +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_init.py:init():623] setting up manager +2024-12-31 23:59:53,327 INFO MainThread:1772311 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 23:59:53,328 INFO MainThread:1772311 [wandb_init.py:init():631] backend started and connected +2024-12-31 23:59:53,341 INFO MainThread:1772311 [wandb_init.py:init():720] updated telemetry +2024-12-31 23:59:53,351 INFO MainThread:1772311 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 23:59:53,651 INFO MainThread:1772311 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 23:59:53,687 INFO MainThread:1772311 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 23:59:53,687 INFO MainThread:1772311 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 23:59:54,062 INFO MainThread:1772311 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 23:59:54,062 INFO MainThread:1772311 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 23:59:54,063 INFO MainThread:1772311 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 23:59:54,063 INFO MainThread:1772311 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 23:59:54,064 INFO MainThread:1772311 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 23:59:54,064 INFO MainThread:1772311 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/tb, True +2024-12-31 23:59:58,426 INFO MainThread:1772311 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-01 00:05:54,484 WARNING MsgRouterThr:1772311 [router.py:message_loop():77] message_loop has been closed diff --git a/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/code/FISH/eval_robot.py b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/config.yaml b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4996dadd78f8dcb30831dc3ee5ed9228e7f564f4 --- /dev/null +++ b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1006_0' +wandb_notes: + desc: null + value: 1006_0_req_1352_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735707593 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/diff.patch b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..625cf877496560323ca22cd6b0a075ba511a2dc6 --- /dev/null +++ b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/diff.patch @@ -0,0 +1,106 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..01bdd86 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1007_0' ++# wandb_run_id: '1002_0' + ++# all books, 6/20 demos per book ++# RGBD+mask+act history ++# wandb_run_id: '1007_0' ++ ++# RGBD+mask+contact(all ftrs)+act history ++wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..50493fe 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -490,7 +490,8 @@ class Workspace: + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] +- total_num_episodes += filtered_run.summary_metrics['episode'] ++ # total_num_episodes += filtered_run.summary_metrics['episode'] ++ total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes +@@ -513,7 +514,6 @@ class Workspace: + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + +- + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index e3031af..6f97e26 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,11 +330,11 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.01 +- self.move_to_rot_tolerance = 2.0 ++ self.move_to_trans_tolerance = 0.005 ++ self.move_to_rot_tolerance = 1.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) +- self.move_to_target_timeout = 15.0 ++ self.move_to_target_timeout = 5.0 + self.move_to_target_max_rotation = 25.0 + # only can use one of these at a time + # assert use_roll + use_pitch + use_yaw <= 1, "Can only use one of roll, pitch, yaw" diff --git a/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table/eval/total_success_rate_ci_6_b6adbd01d0fdba7c5d33.table.json b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table/eval/total_success_rate_ci_6_b6adbd01d0fdba7c5d33.table.json new file mode 100644 index 0000000000000000000000000000000000000000..fa3dc1a8f1f91fd8e58969a66da143eb27da46cc --- /dev/null +++ b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table/eval/total_success_rate_ci_6_b6adbd01d0fdba7c5d33.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[15, 25, 0.6, 0.36213836076632666, 0.8055105953797624, 0.41002793611895416, 0.7874455153367217, 0.439910203230478, 0.744337528895148, 0.4589880406938561, 0.7311604544058057]]} \ No newline at end of file diff --git a/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/0_eval_0_bb2664eb51731886640b.mp4 b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/0_eval_0_bb2664eb51731886640b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7a5f8802b06c6b4ec8f8d40b9e085c4be208491b --- /dev/null +++ b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/0_eval_0_bb2664eb51731886640b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb2664eb51731886640bdaa2f28ef878fb994a6fb7b50cd6e63c1396b1496bc4 +size 1470956 diff --git a/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/1_eval_1_5acda3cc6f6a308de072.mp4 b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/1_eval_1_5acda3cc6f6a308de072.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df0a06a61d16765bfa2bccb4ec05d4f8a7947a4a --- /dev/null +++ b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/1_eval_1_5acda3cc6f6a308de072.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5acda3cc6f6a308de072312035849b844cad8f518137c51f1d20363772ff66a0 +size 1191997 diff --git a/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/2_eval_2_ae958bbddc0d8e506e3a.mp4 b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/2_eval_2_ae958bbddc0d8e506e3a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..89fbb6b8dddad7781185c730c9db71a17bbd7d01 --- /dev/null +++ b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/2_eval_2_ae958bbddc0d8e506e3a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae958bbddc0d8e506e3adfb357cd37bb6d3cc9284e0b259a0cd78c092d04b294 +size 930968 diff --git a/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/3_eval_3_ca451a47ae860c604497.mp4 b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/3_eval_3_ca451a47ae860c604497.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4bcd5c16b64542191693ce55c3b0d99363e9a3aa --- /dev/null +++ b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/3_eval_3_ca451a47ae860c604497.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca451a47ae860c60449746fb81d545cf8171ef7ecde64b9b7d9b3b0a8544de94 +size 1183575 diff --git a/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/4_eval_4_0f9f6705ea3bb2da7eeb.mp4 b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/4_eval_4_0f9f6705ea3bb2da7eeb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a430dfac9d86361168f1c84cb15f50573ab65b76 --- /dev/null +++ b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/4_eval_4_0f9f6705ea3bb2da7eeb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f9f6705ea3bb2da7eebe4e623b267c259b890d89f31660bf2dda08db8e8dbf3 +size 753562 diff --git a/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..b69211c893dc4d8dcf7cbaf9b36fd619c2375067 --- /dev/null +++ b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log @@ -0,0 +1,307 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735707598.426620]: resetting environment +[INFO] [1735707598.431328]: cleared current plan +[INFO] [1735707598.432110]: moving to home +Move to target timeout: 5.000743627548218. Trans error: 0.005461537279188633. Rot error: 0.5529829015319107 +Move to target timeout: 5.101405382156372. Trans error: 0.005396228119924384. Rot error: 1.2693862835505811 +[INFO] [1735707609.537094]: reached home +[INFO] [1735707609.538944]: reset action history +[INFO] [1735707612.248598]: environment reset +Starting episode 0 +[INFO] [1735707612.253273]: resetting environment +[INFO] [1735707612.257773]: cleared current plan +[INFO] [1735707612.258609]: moving to home +Move to target timeout: 5.000591993331909. Trans error: 0.0054145839239347635. Rot error: 1.270635698033093 +[INFO] [1735707618.261268]: reached home +[INFO] [1735707618.262537]: reset action history +[INFO] [1735707620.972446]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735707631.193745]: resetting environment +[INFO] [1735707631.197996]: cleared current plan +[INFO] [1735707631.198915]: moving to home +Move to target timeout: 5.000593900680542. Trans error: 0.00541718862951026. Rot error: 1.2707411116458296 +[INFO] [1735707637.201270]: reached home +[INFO] [1735707637.202854]: reset action history +[INFO] [1735707639.912584]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/episode_rosbags/episode_0_2025-01-01-00-00-39.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1772311.log +started roslaunch server http://158.130.50.37:36875/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1772729] +process[print_text-2]: started with pid [1772753] +process[rosbag_record-3]: started with pid [1772754] +started bagging! +For topic gripper_width: timestamp difference is -101701895 for nearest: 1735707641253268799 - target: 1735707641354970694 at idx 22 +For topic gripper_width: timestamp difference is 105204480 for nearest: 1735707649919938224 - target: 1735707649814733744 at idx 141 +For topic gripper_width: timestamp difference is 106247065 for nearest: 1735707651219937918 - target: 1735707651113690853 at idx 159 +[INFO] [1735707675.114281]: Storing episode... +[print_text-2] killing on exit +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[INFO] [1735707675.859458]: Stored episode 1. +[INFO] [1735707675.861293]: Saving video... +[INFO] [1735707676.052946]: Video saved! +Starting episode 1 +[INFO] [1735707676.063211]: resetting environment +[INFO] [1735707676.071268]: cleared current plan +[INFO] [1735707676.071512]: moving to home +Move to target timeout: 5.001063585281372. Trans error: 0.007786670699715614. Rot error: 1.2183107536972586 +Move to target timeout: 5.099570035934448. Trans error: 0.005623215710013362. Rot error: 0.9364236733320512 +[INFO] [1735707687.174297]: reached home +[INFO] [1735707687.175816]: reset action history +[INFO] [1735707689.886996]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735707693.020596]: resetting environment +[INFO] [1735707693.023771]: cleared current plan +[INFO] [1735707693.023938]: moving to home +Move to target timeout: 5.000657558441162. Trans error: 0.005505580920726061. Rot error: 0.6072932735236388 +Move to target timeout: 5.099775791168213. Trans error: 0.005438863980685711. Rot error: 1.324349617533121 +[INFO] [1735707704.126502]: reached home +[INFO] [1735707704.127376]: reset action history +[INFO] [1735707706.834571]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/episode_rosbags/episode_1_2025-01-01-00-01-46.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1772311.log +started roslaunch server http://158.130.50.37:45301/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1772880] +process[print_text-5]: started with pid [1772881] +process[rosbag_record-6]: started with pid [1772905] +started bagging! +For topic gripper_width: timestamp difference is 118319999 for nearest: 1735707714619936715 - target: 1735707714501616716 at idx 107 +For topic gripper_width: timestamp difference is 101957178 for nearest: 1735707722919968939 - target: 1735707722818011761 at idx 216 +For topic gripper_width: timestamp difference is 104845240 for nearest: 1735707732019962742 - target: 1735707731915117502 at idx 249 +preempting episode +[INFO] [1735707734.278812]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735707734.995467]: Stored episode 2. +[INFO] [1735707734.997144]: Saving video... +[INFO] [1735707735.660753]: Video saved! +Starting episode 2 +[INFO] [1735707735.695422]: resetting environment +[INFO] [1735707735.713563]: cleared current plan +[INFO] [1735707735.713810]: moving to home +[WARN] [1735707735.441438]: Plan exhausted +[WARN] [1735707735.482760]: Plan exhausted +[WARN] [1735707735.522057]: Plan exhausted +[WARN] [1735707735.561737]: Plan exhausted +[WARN] [1735707735.601569]: Plan exhausted +[WARN] [1735707735.645131]: Plan exhausted +[WARN] [1735707735.681445]: Plan exhausted +Move to target timeout: 5.101730585098267. Trans error: 0.0056270285244559. Rot error: 0.9711638293786591 +[INFO] [1735707746.121067]: reached home +[INFO] [1735707746.122935]: reset action history +[INFO] [1735707748.916378]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735707761.944001]: resetting environment +[INFO] [1735707761.954209]: cleared current plan +[INFO] [1735707761.954407]: moving to home +[INFO] [1735707762.955317]: reached home +[INFO] [1735707762.955936]: reset action history +[INFO] [1735707765.714968]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/episode_rosbags/episode_2_2025-01-01-00-02-45.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1772311.log +started roslaunch server http://158.130.50.37:37947/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1773036] +process[print_text-8]: started with pid [1773037] +process[rosbag_record-9]: started with pid [1773061] +started bagging! +For topic gripper_width: timestamp difference is -233784481 for nearest: 1735707766453245596 - target: 1735707766687030077 at idx 9 +preempting episode +[INFO] [1735707786.490530]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1735707787.221338]: Stored episode 3. +[INFO] [1735707787.227138]: Saving video... +[INFO] [1735707787.675142]: Video saved! +Starting episode 3 +[INFO] [1735707787.720887]: resetting environment +[INFO] [1735707787.733851]: cleared current plan +[INFO] [1735707787.734084]: moving to home +Move to target timeout: 5.002325773239136. Trans error: 0.005684637930244207. Rot error: 0.60360866425378 +Move to target timeout: 5.094324588775635. Trans error: 0.005558751914911254. Rot error: 0.9367403977252872 +[INFO] [1735707798.848376]: reached home +[INFO] [1735707798.852987]: reset action history +[INFO] [1735707801.637499]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735707815.821543]: resetting environment +[INFO] [1735707815.833609]: cleared current plan +[INFO] [1735707815.834459]: moving to home +Move to target timeout: 5.001443147659302. Trans error: 0.005486711394041777. Rot error: 0.6050148635587148 +Move to target timeout: 5.111356496810913. Trans error: 0.005465575395154654. Rot error: 1.3056768123284208 +[INFO] [1735707826.958425]: reached home +[INFO] [1735707826.960806]: reset action history +[INFO] [1735707829.685622]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/episode_rosbags/episode_3_2025-01-01-00-03-49.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1772311.log +started roslaunch server http://158.130.50.37:34999/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1773189] +process[print_text-11]: started with pid [1773190] +process[rosbag_record-12]: started with pid [1773214] +started bagging! +preempting episode +[INFO] [1735707856.994351]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1735707857.745682]: Stored episode 4. +[INFO] [1735707857.746308]: Saving video... +[WARN] [1735707858.234917]: Plan exhausted +[WARN] [1735707858.274822]: Plan exhausted +[WARN] [1735707858.314424]: Plan exhausted +[WARN] [1735707858.362331]: Plan exhausted +[WARN] [1735707858.398771]: Plan exhausted +[WARN] [1735707858.440612]: Plan exhausted +[WARN] [1735707858.476273]: Plan exhausted +[WARN] [1735707858.514814]: Plan exhausted +[INFO] [1735707858.460344]: Video saved! +Starting episode 4 +[INFO] [1735707858.507436]: resetting environment +[INFO] [1735707858.529855]: cleared current plan +[INFO] [1735707858.530273]: moving to home +Move to target timeout: 5.0030927658081055. Trans error: 0.007933398708701134. Rot error: 1.3106648888865144 +Move to target timeout: 5.097123384475708. Trans error: 0.0055648240548801. Rot error: 0.9584039580987723 +[INFO] [1735707869.639250]: reached home +[INFO] [1735707869.645496]: reset action history +[INFO] [1735707872.424510]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735707883.752036]: resetting environment +[INFO] [1735707883.779443]: cleared current plan +[INFO] [1735707883.785135]: moving to home +Move to target timeout: 5.000685214996338. Trans error: 0.0055054775439202785. Rot error: 0.6234144264261202 +Move to target timeout: 5.099234104156494. Trans error: 0.005468493430529113. Rot error: 1.3298218283714223 +[INFO] [1735707894.894468]: reached home +[INFO] [1735707894.896596]: reset action history +[INFO] [1735707897.675714]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/episode_rosbags/episode_4_2025-01-01-00-04-57.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1772311.log +started roslaunch server http://158.130.50.37:33217/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1773345] +process[print_text-14]: started with pid [1773346] +process[rosbag_record-15]: started with pid [1773370] +started bagging! +For topic gripper_width: timestamp difference is 107979864 for nearest: 1735707901019963831 - target: 1735707900911983967 at idx 53 +preempting episode +[INFO] [1735707914.089481]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1735707914.931807]: Stored episode 5. +[INFO] [1735707914.937692]: Saving video... +[INFO] [1735707915.225610]: Video saved! +[INFO] [1735707915.265445]: resetting environment +[INFO] [1735707915.299405]: cleared current plan +[INFO] [1735707915.303237]: moving to home +Move to target timeout: 5.000735282897949. Trans error: 0.007653213106095791. Rot error: 1.227663926082395 +Move to target timeout: 5.098784446716309. Trans error: 0.005616603494278466. Rot error: 0.9934581834176512 +[INFO] [1735707926.408889]: reached home +[INFO] [1735707926.417067]: reset action history +[INFO] [1735707929.219775]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/requirements.txt b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-metadata.json b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..08dae164f9ba815f9df851acd61fc9e782645d44 --- /dev/null +++ b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T04:59:53.754835", + "startedAt": "2025-01-01T04:59:53.320075", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3562.501, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.987, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.045, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 632.6411437988281 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..545ae9d203dd8df8fd22e7e2b6d6eb20b0ea12a2 --- /dev/null +++ b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "bb2664eb51731886640bdaa2f28ef878fb994a6fb7b50cd6e63c1396b1496bc4", "size": 1470956, "path": "media/videos/eval/0_eval_0_bb2664eb51731886640b.mp4"}, "global_step": 482, "_timestamp": 1735707938.8092089, "_runtime": 345.480673789978, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "5acda3cc6f6a308de072312035849b844cad8f518137c51f1d20363772ff66a0", "size": 1191997, "path": "media/videos/eval/1_eval_1_5acda3cc6f6a308de072.mp4"}, "eval/num_success": 0.0, "episode": 25.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "ae958bbddc0d8e506e3adfb357cd37bb6d3cc9284e0b259a0cd78c092d04b294", "size": 930968, "path": "media/videos/eval/2_eval_2_ae958bbddc0d8e506e3a.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "ca451a47ae860c60449746fb81d545cf8171ef7ecde64b9b7d9b3b0a8544de94", "size": 1183575, "path": "media/videos/eval/3_eval_3_ca451a47ae860c604497.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "0f9f6705ea3bb2da7eebe4e623b267c259b890d89f31660bf2dda08db8e8dbf3", "size": 753562, "path": "media/videos/eval/4_eval_4_0f9f6705ea3bb2da7eeb.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://3adbxibakhv3ei7bo6yl5g2pc88bkg08wltw5hurzvuux6du4q08gqfc7jedjacxtys1y9qihptzo548d5ynyuaaarfhwjwdibeff9y574dla89zy8jfw8s5oy4e6x16/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://572nse38ygzfmwviwf1ptmv7mpsdfl5s070g6b5hxuiq4od6xqbidqqmo1boqn89etr686zq1gutsjbz4yx2559axenbfqoa8s6mtqzidrp2gd6glnpfz9t3s5ws5q8e:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 15, "total_num_episodes": 25, "total_success_rate": 0.6, "total_success_rate_lb_0.01": 0.36213836076632666, "total_success_rate_ub_0.01": 0.8055105953797624, "total_success_rate_lb_0.025": 0.41002793611895416, "total_success_rate_ub_0.025": 0.7874455153367217, "total_success_rate_lb_0.05": 0.439910203230478, "total_success_rate_ub_0.05": 0.744337528895148, "total_success_rate_lb_0.1": 0.4589880406938561, "total_success_rate_ub_0.1": 0.7311604544058057, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "b6adbd01d0fdba7c5d33ec099346a9e8bb28c5502d3a41d27afc2be18f571735", "size": 505, "artifact_path": "wandb-client-artifact://pe2wl1yg9q6apbuehe5b01gv585dp84km7o4k2bhiihbvfyspb57j5v9cyyg6c8v04jdp8iu4tmu7vddippbmjjra7mi194n90yb8ydmhd8g1z4dzshj14kolpxlcrp2/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://9vwdhu8m2r4sdqicanhf9i3bcad4mztcj8lczes2qwxjyxmgvt1ygfb3w4arkxswi66xym4dnplxjmgmrnd9gdi6kt17nz0tpthb2kglsrvmz6rtvngib9cb8qld35xi:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_b6adbd01d0fdba7c5d33.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 348}, "eval/total_success_rate": 0.6000000238418579, "eval/total_success_rate_lb_0.01": 0.3621383607387543, "eval/total_success_rate_ub_0.01": 0.8055105805397034, "eval/total_success_rate_lb_0.025": 0.41002795100212097, "eval/total_success_rate_ub_0.025": 0.7874454855918884, "eval/total_success_rate_lb_0.05": 0.4399102032184601, "eval/total_success_rate_ub_0.05": 0.7443374991416931, "eval/total_success_rate_lb_0.1": 0.4589880406856537, "eval/total_success_rate_ub_0.1": 0.7311604619026184} \ No newline at end of file diff --git a/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/logs/debug-internal.log b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5cdb244818596d6e6f712691f2295df29314a2fd --- /dev/null +++ b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/logs/debug-internal.log @@ -0,0 +1,902 @@ +2024-12-31 23:59:53,328 INFO StreamThr :1772506 [internal.py:wandb_internal():85] W&B internal server running at pid: 1772506, started at: 2024-12-31 23:59:53.328438 +2024-12-31 23:59:53,330 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status +2024-12-31 23:59:53,342 INFO WriterThread:1772506 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/run-5r8mx4bu.wandb +2024-12-31 23:59:53,342 DEBUG SenderThread:1772506 [sender.py:send():379] send: header +2024-12-31 23:59:53,354 DEBUG SenderThread:1772506 [sender.py:send():379] send: run +2024-12-31 23:59:53,633 INFO SenderThread:1772506 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files +2024-12-31 23:59:53,633 INFO SenderThread:1772506 [sender.py:_start_run_threads():1188] run started: 5r8mx4bu with start time 1735707593.328535 +2024-12-31 23:59:53,651 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 23:59:53,652 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: check_version +2024-12-31 23:59:53,737 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 23:59:53,745 DEBUG HandlerThread:1772506 [system_info.py:__init__():26] System info init +2024-12-31 23:59:53,745 DEBUG HandlerThread:1772506 [system_info.py:__init__():41] System info init done +2024-12-31 23:59:53,745 INFO HandlerThread:1772506 [system_monitor.py:start():194] Starting system monitor +2024-12-31 23:59:53,745 INFO SystemMonitor:1772506 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 23:59:53,745 INFO HandlerThread:1772506 [system_monitor.py:probe():214] Collecting system info +2024-12-31 23:59:53,746 INFO SystemMonitor:1772506 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 23:59:53,747 INFO SystemMonitor:1772506 [interfaces.py:start():188] Started disk monitoring +2024-12-31 23:59:53,748 INFO SystemMonitor:1772506 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 23:59:53,748 INFO SystemMonitor:1772506 [interfaces.py:start():188] Started memory monitoring +2024-12-31 23:59:53,749 INFO SystemMonitor:1772506 [interfaces.py:start():188] Started network monitoring +2024-12-31 23:59:53,754 DEBUG HandlerThread:1772506 [system_info.py:probe():152] Probing system +2024-12-31 23:59:53,756 DEBUG HandlerThread:1772506 [system_info.py:_probe_git():137] Probing git +2024-12-31 23:59:53,761 DEBUG HandlerThread:1772506 [system_info.py:_probe_git():145] Probing git done +2024-12-31 23:59:53,761 DEBUG HandlerThread:1772506 [system_info.py:probe():200] Probing system done +2024-12-31 23:59:53,761 DEBUG HandlerThread:1772506 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T04:59:53.754835', 'startedAt': '2025-01-01T04:59:53.320075', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.501, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.987, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.045, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 632.6411437988281}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 23:59:53,761 INFO HandlerThread:1772506 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 23:59:53,761 INFO HandlerThread:1772506 [system_monitor.py:probe():227] Publishing system info +2024-12-31 23:59:53,761 DEBUG HandlerThread:1772506 [system_info.py:_save_code():44] Saving code +2024-12-31 23:59:53,763 DEBUG HandlerThread:1772506 [system_info.py:_save_code():67] Saving code done +2024-12-31 23:59:53,763 DEBUG HandlerThread:1772506 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 23:59:53,824 DEBUG HandlerThread:1772506 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 23:59:53,824 INFO HandlerThread:1772506 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 23:59:53,828 DEBUG SenderThread:1772506 [sender.py:send():379] send: files +2024-12-31 23:59:53,828 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 23:59:53,828 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 23:59:53,828 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 23:59:54,061 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 23:59:54,062 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: python_packages +2024-12-31 23:59:54,062 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/z3trdkb9-wandb-metadata.json +2024-12-31 23:59:54,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:59:54,063 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:59:54,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:59:54,065 INFO HandlerThread:1772506 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/tb" + save: true +} +control { + relay_id: "b88240e5503a46e9a3ae165fd55e2f33" +} +_info { + stream_id: "5r8mx4bu" +} + +2024-12-31 23:59:54,065 DEBUG HandlerThread:1772506 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 23:59:54,106 INFO wandb-upload_2:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/39i0rf51-diff.patch +2024-12-31 23:59:54,145 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/m6vzfnnh-code/FISH/eval_robot.py +2024-12-31 23:59:54,170 DEBUG SenderThread:1772506 [sender.py:send():379] send: telemetry +2024-12-31 23:59:54,170 DEBUG SenderThread:1772506 [sender.py:send():379] send: tbrecord +2024-12-31 23:59:54,171 DEBUG SenderThread:1772506 [sender.py:send():379] send: files +2024-12-31 23:59:54,171 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 with policy live +2024-12-31 23:59:54,368 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/4p2z9ne0-events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2024-12-31 23:59:54,635 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/code/FISH/eval_robot.py +2024-12-31 23:59:54,635 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/requirements.txt +2024-12-31 23:59:54,635 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2024-12-31 23:59:54,635 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2024-12-31 23:59:54,635 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/diff.patch +2024-12-31 23:59:54,636 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-metadata.json +2024-12-31 23:59:54,636 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/code/FISH +2024-12-31 23:59:54,636 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/code +2024-12-31 23:59:55,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:59:56,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:59:56,634 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2024-12-31 23:59:57,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:59:58,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:59:58,426 DEBUG SenderThread:1772506 [sender.py:send():379] send: config +2024-12-31 23:59:58,426 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:59:59,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:00,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:00,635 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:01,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:02,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:03,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:03,434 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:04,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:04,637 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:05,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:06,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:07,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:08,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:08,536 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:09,062 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:00:09,062 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:00:09,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:10,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:10,639 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:11,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:12,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:13,065 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:14,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:14,065 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:14,641 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:15,065 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:16,065 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:17,065 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:18,065 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:18,642 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:19,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:19,066 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:20,066 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:20,642 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:21,066 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:22,066 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:22,643 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:23,066 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:24,062 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:00:24,062 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:00:24,105 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:24,200 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:24,643 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/config.yaml +2025-01-01 00:00:25,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:26,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:27,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:28,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:29,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:30,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:30,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:31,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:32,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:32,646 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:33,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:34,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:35,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:35,068 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:36,068 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:37,068 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:38,068 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:38,647 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:39,062 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:00:39,062 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:00:39,105 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:40,066 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:40,068 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:41,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:42,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:42,648 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:43,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:44,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:45,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:46,064 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:46,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:47,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:48,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:49,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:50,071 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:51,071 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:51,225 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:52,071 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:52,651 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:00:53,071 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:53,749 DEBUG SystemMonitor:1772506 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 00:00:53,750 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:00:54,062 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:00:54,062 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:00:54,105 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:55,071 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:56,071 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:57,071 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:57,146 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:00:58,072 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:00:59,072 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:00,072 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:01,072 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:02,072 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:03,065 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:03,072 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:04,072 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:05,072 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:06,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:07,072 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:08,065 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:08,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:09,062 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:01:09,062 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:01:09,109 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:10,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:11,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:12,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:13,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:13,198 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:14,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:15,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:16,062 DEBUG SenderThread:1772506 [sender.py:send():379] send: files +2025-01-01 00:01:16,062 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_bb2664eb51731886640b.mp4 with policy now +2025-01-01 00:01:16,062 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:01:16,063 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:01:16,063 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:01:16,065 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:01:16,075 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:16,330 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/0bwxxs12-media/videos/eval/0_eval_0_bb2664eb51731886640b.mp4 +2025-01-01 00:01:16,658 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:16,658 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:01:16,659 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/0_eval_0_bb2664eb51731886640b.mp4 +2025-01-01 00:01:16,659 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media +2025-01-01 00:01:16,659 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval +2025-01-01 00:01:16,659 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos +2025-01-01 00:01:16,933 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/0r8fzcd1-media/videos/eval/0_eval_0_bb2664eb51731886640b.mp4 +2025-01-01 00:01:17,074 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:18,076 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:18,659 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:19,065 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:19,075 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:20,074 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:21,113 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:22,078 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:22,660 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:23,078 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:23,750 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:01:24,062 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:01:24,062 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:01:24,105 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:24,173 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:25,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:26,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:27,078 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:28,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:28,662 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:29,078 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:29,887 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:30,078 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:31,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:32,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:32,663 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:33,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:33,664 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:01:33,930 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/bt73oq8c-events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:01:34,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:34,664 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:35,066 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:35,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:36,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:37,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:38,079 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:39,062 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:01:39,063 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:01:39,109 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:40,080 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:40,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:40,666 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:41,080 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:42,080 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:43,080 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:44,080 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:44,668 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:45,080 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:45,169 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:46,080 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:46,668 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:47,081 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:48,081 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:48,669 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:49,081 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:50,081 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:51,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:51,082 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:52,081 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:53,081 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:53,751 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:01:54,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:01:54,063 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:01:54,105 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:55,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:56,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:56,669 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:01:56,672 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:01:57,082 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:58,082 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:01:59,082 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:00,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:01,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:02,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:02,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:03,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:04,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:04,674 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:05,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:06,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:07,067 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:07,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:08,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:09,063 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:02:09,063 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:02:09,105 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:10,084 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:11,084 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:12,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:12,084 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:13,084 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:14,084 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:14,677 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:15,084 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:15,678 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/1_eval_1_5acda3cc6f6a308de072.mp4 +2025-01-01 00:02:15,678 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval +2025-01-01 00:02:15,684 DEBUG SenderThread:1772506 [sender.py:send():379] send: files +2025-01-01 00:02:15,684 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_5acda3cc6f6a308de072.mp4 with policy now +2025-01-01 00:02:15,685 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:02:15,686 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:02:15,686 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:02:15,686 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:02:15,941 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/go50pzf2-media/videos/eval/1_eval_1_5acda3cc6f6a308de072.mp4 +2025-01-01 00:02:16,085 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:16,209 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/zxu8ovf9-media/videos/eval/1_eval_1_5acda3cc6f6a308de072.mp4 +2025-01-01 00:02:16,678 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:16,678 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:02:17,087 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:17,678 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:17,714 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:18,090 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:19,085 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:20,093 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:21,094 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:22,091 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:23,075 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:23,093 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:23,751 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:02:24,068 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:02:24,068 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:02:24,113 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:25,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:26,097 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:26,681 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:27,101 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:28,098 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:28,165 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:28,682 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:29,095 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:30,095 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:30,683 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:31,095 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:32,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:33,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:34,069 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:34,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:35,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:36,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:37,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:38,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:39,066 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:02:39,066 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:02:39,109 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:39,181 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:40,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:41,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:42,096 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:42,631 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:02:42,632 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:02:42,632 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:02:42,686 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:02:42,687 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:02:42,938 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/ptwszs2x-events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:02:43,097 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:44,097 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:44,687 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:45,069 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:45,097 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:46,098 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:46,688 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:47,098 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:48,098 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:48,689 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:02:49,099 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:50,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:50,101 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:51,100 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:52,102 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:53,102 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:53,752 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:02:54,066 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:02:54,066 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:02:54,109 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:55,102 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:55,188 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:02:56,102 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:57,102 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:58,103 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:02:59,103 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:00,103 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:01,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:01,103 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:02,103 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:03,103 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:04,104 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:05,104 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:06,070 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:06,104 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:07,105 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:07,713 DEBUG SenderThread:1772506 [sender.py:send():379] send: files +2025-01-01 00:03:07,713 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_ae958bbddc0d8e506e3a.mp4 with policy now +2025-01-01 00:03:07,716 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:03:07,716 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:03:07,716 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:03:07,717 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:03:07,965 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/81v92fm9-media/videos/eval/2_eval_2_ae958bbddc0d8e506e3a.mp4 +2025-01-01 00:03:08,104 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:08,695 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:08,696 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:03:08,696 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/2_eval_2_ae958bbddc0d8e506e3a.mp4 +2025-01-01 00:03:08,696 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval +2025-01-01 00:03:09,006 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/qy3vxjrr-media/videos/eval/2_eval_2_ae958bbddc0d8e506e3a.mp4 +2025-01-01 00:03:09,081 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:03:09,081 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:03:09,125 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:10,109 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:11,113 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:11,226 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:12,114 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:13,111 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:14,120 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:14,698 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:15,116 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:16,119 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:17,082 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:17,117 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:18,126 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:18,699 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:19,121 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:20,130 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:20,700 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:21,123 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:22,122 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:22,685 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:22,700 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:23,125 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:23,752 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:03:24,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:03:24,073 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:03:24,127 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:25,123 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:26,125 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:27,124 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:28,127 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:28,210 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:29,131 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:30,131 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:31,128 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:32,131 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:33,131 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:34,081 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:34,130 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:35,130 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:36,130 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:36,399 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:03:36,399 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:03:36,399 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:03:36,705 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:36,705 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:03:36,705 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:03:37,132 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:38,132 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:39,074 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:03:39,074 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:03:39,132 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:39,231 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:40,136 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:41,133 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:42,138 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:42,707 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:43,142 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:44,139 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:45,084 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:45,139 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:46,138 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:47,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:48,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:48,709 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:49,146 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:50,140 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:50,710 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:50,950 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:51,140 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:52,140 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:52,711 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:03:53,140 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:53,752 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:03:54,069 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:03:54,070 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:03:54,141 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:55,141 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:56,129 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:03:56,141 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:57,141 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:58,141 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:03:59,142 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:00,143 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:01,142 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:02,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:02,142 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:03,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:04,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:05,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:06,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:07,083 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:07,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:08,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:09,071 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:04:09,071 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:04:09,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:10,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:11,144 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:12,145 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:12,192 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:13,145 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:14,145 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:15,145 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:16,145 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:17,145 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:17,746 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:18,153 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:18,501 DEBUG SenderThread:1772506 [sender.py:send():379] send: files +2025-01-01 00:04:18,502 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ca451a47ae860c604497.mp4 with policy now +2025-01-01 00:04:18,505 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:04:18,506 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:04:18,506 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:04:18,506 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:04:18,720 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:18,720 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:04:18,720 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/3_eval_3_ca451a47ae860c604497.mp4 +2025-01-01 00:04:18,720 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval +2025-01-01 00:04:18,786 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/vjpaulnw-media/videos/eval/3_eval_3_ca451a47ae860c604497.mp4 +2025-01-01 00:04:19,075 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/4r8q538n-media/videos/eval/3_eval_3_ca451a47ae860c604497.mp4 +2025-01-01 00:04:19,156 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:19,720 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:20,156 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:20,721 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:21,156 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:22,151 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:23,087 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:23,155 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:23,753 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:04:24,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:04:24,073 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:04:24,162 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:24,722 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:25,162 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:26,158 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:27,158 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:28,162 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:28,187 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:29,162 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:30,165 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:30,724 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:31,170 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:32,171 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:33,168 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:33,469 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:34,167 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:34,725 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:35,171 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:36,167 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:37,173 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:38,172 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:39,077 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:04:39,078 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:04:39,173 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:39,223 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:40,172 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:41,172 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:42,170 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:43,173 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:43,728 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:04:43,997 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/05clmi3g-events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:04:44,172 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:44,482 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:04:44,483 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:04:44,483 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:44,484 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:04:44,729 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:44,729 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:04:44,729 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:04:45,176 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:46,178 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:47,176 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:48,180 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:49,178 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:50,090 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:50,186 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:50,731 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:51,178 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:52,184 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:53,181 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:53,753 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:04:54,075 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:04:54,075 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:04:54,179 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:54,732 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:55,183 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:55,937 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:04:56,184 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:56,733 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:57,183 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:58,182 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:04:58,733 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:04:59,182 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:00,182 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:01,021 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:01,182 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:02,182 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:02,735 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:03,182 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:04,182 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:05,183 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:06,089 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:06,183 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:07,183 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:08,183 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:09,073 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:05:09,073 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:05:09,183 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:10,183 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:11,183 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:11,195 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:12,184 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:13,184 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:14,221 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:15,194 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:15,273 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:05:15,273 DEBUG SenderThread:1772506 [sender.py:send():379] send: files +2025-01-01 00:05:15,273 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_0f9f6705ea3bb2da7eeb.mp4 with policy now +2025-01-01 00:05:15,274 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:05:15,274 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:15,277 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:15,512 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/06qmqt7h-media/videos/eval/4_eval_4_0f9f6705ea3bb2da7eeb.mp4 +2025-01-01 00:05:15,739 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:05:15,740 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/4_eval_4_0f9f6705ea3bb2da7eeb.mp4 +2025-01-01 00:05:15,740 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval +2025-01-01 00:05:15,991 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/y8po8ayg-media/videos/eval/4_eval_4_0f9f6705ea3bb2da7eeb.mp4 +2025-01-01 00:05:16,204 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:16,304 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:16,740 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:17,203 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:18,194 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:19,198 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:20,203 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:21,197 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:21,345 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:22,198 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:22,742 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:23,200 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:23,754 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:05:24,087 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:05:24,087 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:05:24,221 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:25,203 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:26,206 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:26,410 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:26,743 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:27,208 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:28,214 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:28,744 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:29,216 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:30,213 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:30,744 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:31,218 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:32,214 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:32,261 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:33,211 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:34,225 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:35,215 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:36,219 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:37,216 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:37,865 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:38,221 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:38,298 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 00:05:38,298 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 00:05:38,483 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:05:38,747 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:38,747 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:05:38,748 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 00:05:38,748 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media +2025-01-01 00:05:38,748 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table +2025-01-01 00:05:38,748 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table/eval +2025-01-01 00:05:38,785 INFO wandb-upload_0:1772506 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmdq_ekj4 +2025-01-01 00:05:38,886 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 00:05:38,895 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 00:05:38,938 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/5038he12-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 00:05:39,080 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 00:05:39,213 INFO SenderThread:1772506 [sender.py:send_request_log_artifact():1525] logged artifact run-5r8mx4bu-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjgzNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE4MDQ1MA==', 'latestArtifact': None}} +2025-01-01 00:05:39,213 DEBUG SenderThread:1772506 [sender.py:send():379] send: files +2025-01-01 00:05:39,213 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-01 00:05:39,213 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:05:39,213 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,214 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,214 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:05:39,214 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,214 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,215 DEBUG SenderThread:1772506 [sender.py:send():379] send: telemetry +2025-01-01 00:05:39,215 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,215 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,215 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,216 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,216 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:39,216 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,218 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,218 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,218 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,219 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,219 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,219 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,219 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,219 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,220 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,220 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,220 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,220 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,221 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,221 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,221 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,221 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:39,222 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:39,222 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 00:05:39,457 INFO wandb-upload_2:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/uqpqat07-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 00:05:39,665 INFO wandb-upload_1:1772506 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpq34rapwq +2025-01-01 00:05:39,749 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:05:39,749 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:05:39,749 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table/eval/total_success_rate_ci_6_b6adbd01d0fdba7c5d33.table.json +2025-01-01 00:05:39,749 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table/eval +2025-01-01 00:05:40,096 INFO SenderThread:1772506 [sender.py:send_request_log_artifact():1525] logged artifact run-5r8mx4bu-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjg0NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE4MDQ1MQ==', 'latestArtifact': None}} +2025-01-01 00:05:40,096 DEBUG SenderThread:1772506 [sender.py:send():379] send: files +2025-01-01 00:05:40,096 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_b6adbd01d0fdba7c5d33.table.json with policy now +2025-01-01 00:05:40,097 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:05:40,097 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:40,097 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:40,098 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: stop_status +2025-01-01 00:05:40,215 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:40,276 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/22khlc6s-media/table/eval/total_success_rate_ci_6_b6adbd01d0fdba7c5d33.table.json +2025-01-01 00:05:40,547 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/2kg0vga6-media/table/eval/total_success_rate_ci_6_b6adbd01d0fdba7c5d33.table.json +2025-01-01 00:05:40,749 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:40,750 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:05:41,215 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:42,209 DEBUG SenderThread:1772506 [sender.py:send():379] send: exit +2025-01-01 00:05:42,209 INFO SenderThread:1772506 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 00:05:42,210 INFO SenderThread:1772506 [sender.py:send_exit():588] handling runtime: 348 +2025-01-01 00:05:42,210 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:42,210 INFO SenderThread:1772506 [sender.py:send_exit():594] send defer +2025-01-01 00:05:42,210 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:42,210 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 00:05:42,210 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:42,210 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 00:05:42,210 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 1 +2025-01-01 00:05:42,211 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:42,211 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 00:05:42,211 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:42,211 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 00:05:42,211 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 2 +2025-01-01 00:05:42,211 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:42,211 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 00:05:42,211 INFO HandlerThread:1772506 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 00:05:42,211 DEBUG SystemMonitor:1772506 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 00:05:42,211 DEBUG SystemMonitor:1772506 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 00:05:42,212 INFO HandlerThread:1772506 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 00:05:42,212 INFO HandlerThread:1772506 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 00:05:42,262 INFO HandlerThread:1772506 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 00:05:42,263 INFO HandlerThread:1772506 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 00:05:42,263 INFO HandlerThread:1772506 [interfaces.py:finish():200] Joined network monitor +2025-01-01 00:05:42,263 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:42,263 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 00:05:42,263 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 3 +2025-01-01 00:05:42,263 DEBUG SenderThread:1772506 [sender.py:send():379] send: stats +2025-01-01 00:05:42,264 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:42,264 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 00:05:42,264 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:42,264 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 00:05:42,264 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 4 +2025-01-01 00:05:42,264 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:42,264 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 00:05:42,750 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:05:44,751 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:48,549 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:05:48,549 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:48,549 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 00:05:48,549 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:48,550 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:48,550 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 00:05:48,550 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 5 +2025-01-01 00:05:48,551 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:05:48,551 DEBUG SenderThread:1772506 [sender.py:send():379] send: history +2025-01-01 00:05:48,551 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: summary_record +2025-01-01 00:05:48,551 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:48,551 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 00:05:48,552 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:48,553 DEBUG SenderThread:1772506 [sender.py:send():379] send: summary +2025-01-01 00:05:48,553 INFO SenderThread:1772506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 00:05:48,554 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:48,554 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 00:05:48,554 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 6 +2025-01-01 00:05:48,554 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:48,554 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 00:05:48,554 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:48,554 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 00:05:48,554 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 7 +2025-01-01 00:05:48,554 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 00:05:48,554 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:48,554 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 00:05:48,555 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:48,555 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 00:05:48,753 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/config.yaml +2025-01-01 00:05:48,753 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:05:49,211 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:05:51,645 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 8 +2025-01-01 00:05:51,645 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:05:51,645 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:51,646 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 00:05:51,646 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:51,646 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 00:05:51,646 INFO SenderThread:1772506 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 00:05:51,648 INFO SenderThread:1772506 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 00:05:51,650 INFO SenderThread:1772506 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 00:05:51,663 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 9 +2025-01-01 00:05:51,664 DEBUG SenderThread:1772506 [sender.py:send():379] send: artifact +2025-01-01 00:05:51,667 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:51,667 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 00:05:51,754 INFO Thread-12 :1772506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:52,033 INFO wandb-upload_1:1772506 [upload_job.py:push():85] Skipped uploading /tmp/tmptzi91ge5/wandb-job.json +2025-01-01 00:05:52,033 INFO wandb-upload_2:1772506 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9f8_6z8e +2025-01-01 00:05:52,154 INFO wandb-upload_0:1772506 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmps8pm0g3d +2025-01-01 00:05:52,211 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:05:52,687 INFO SenderThread:1772506 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjk2Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDIxODYyMQ==', 'versionIndex': 402}}} +2025-01-01 00:05:52,687 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:52,687 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 00:05:52,687 INFO SenderThread:1772506 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 00:05:52,755 INFO SenderThread:1772506 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files +2025-01-01 00:05:52,755 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/config.yaml config.yaml +2025-01-01 00:05:52,755 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json wandb-summary.json +2025-01-01 00:05:52,757 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/requirements.txt requirements.txt +2025-01-01 00:05:52,759 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-metadata.json wandb-metadata.json +2025-01-01 00:05:52,759 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:05:52,759 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log output.log +2025-01-01 00:05:52,760 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/diff.patch diff.patch +2025-01-01 00:05:52,761 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 00:05:52,761 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/3_eval_3_ca451a47ae860c604497.mp4 media/videos/eval/3_eval_3_ca451a47ae860c604497.mp4 +2025-01-01 00:05:52,761 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/2_eval_2_ae958bbddc0d8e506e3a.mp4 media/videos/eval/2_eval_2_ae958bbddc0d8e506e3a.mp4 +2025-01-01 00:05:52,761 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/4_eval_4_0f9f6705ea3bb2da7eeb.mp4 media/videos/eval/4_eval_4_0f9f6705ea3bb2da7eeb.mp4 +2025-01-01 00:05:52,761 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/0_eval_0_bb2664eb51731886640b.mp4 media/videos/eval/0_eval_0_bb2664eb51731886640b.mp4 +2025-01-01 00:05:52,761 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/videos/eval/1_eval_1_5acda3cc6f6a308de072.mp4 media/videos/eval/1_eval_1_5acda3cc6f6a308de072.mp4 +2025-01-01 00:05:52,761 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table/eval/total_success_rate_ci_6_b6adbd01d0fdba7c5d33.table.json media/table/eval/total_success_rate_ci_6_b6adbd01d0fdba7c5d33.table.json +2025-01-01 00:05:52,763 INFO SenderThread:1772506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 00:05:52,763 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 10 +2025-01-01 00:05:52,763 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:05:52,763 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:52,763 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 00:05:52,764 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:52,765 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 00:05:52,765 INFO SenderThread:1772506 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 00:05:52,956 INFO wandb-upload_2:1772506 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/requirements.txt +2025-01-01 00:05:52,977 INFO wandb-upload_0:1772506 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/wandb-summary.json +2025-01-01 00:05:52,980 INFO wandb-upload_1:1772506 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/config.yaml +2025-01-01 00:05:53,018 INFO wandb-upload_4:1772506 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/files/output.log +2025-01-01 00:05:53,056 INFO wandb-upload_3:1772506 [upload_job.py:push():130] Uploaded file /tmp/tmpk8rcvxghwandb/k4qf7v3f-events.out.tfevents.1735707594.leonmkim-ROG-Strix-G15CS-G15CS.1772311.0 +2025-01-01 00:05:53,212 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:05:53,212 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:05:53,257 INFO Thread-11 (_thread_body):1772506 [sender.py:transition_state():614] send defer: 11 +2025-01-01 00:05:53,257 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:53,257 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 00:05:53,257 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:53,257 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 00:05:53,258 INFO SenderThread:1772506 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 00:05:53,259 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 12 +2025-01-01 00:05:53,259 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:53,259 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 00:05:53,259 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:53,259 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 00:05:53,259 INFO SenderThread:1772506 [file_stream.py:finish():601] file stream finish called +2025-01-01 00:05:53,423 INFO SenderThread:1772506 [file_stream.py:finish():605] file stream finish is done +2025-01-01 00:05:53,424 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 13 +2025-01-01 00:05:53,424 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:53,424 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 00:05:53,424 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:53,424 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 00:05:53,424 INFO SenderThread:1772506 [sender.py:transition_state():614] send defer: 14 +2025-01-01 00:05:53,424 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: defer +2025-01-01 00:05:53,424 INFO HandlerThread:1772506 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 00:05:53,425 DEBUG SenderThread:1772506 [sender.py:send():379] send: final +2025-01-01 00:05:53,425 DEBUG SenderThread:1772506 [sender.py:send():379] send: footer +2025-01-01 00:05:53,425 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: defer +2025-01-01 00:05:53,425 INFO SenderThread:1772506 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 00:05:53,425 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:05:53,426 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:05:53,426 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 00:05:53,426 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 00:05:53,427 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 00:05:53,428 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 00:05:53,429 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 00:05:53,429 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: server_info +2025-01-01 00:05:53,482 DEBUG SenderThread:1772506 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 00:05:53,482 INFO MainThread:1772506 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 00:05:53,483 INFO MainThread:1772506 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 00:05:53,483 INFO MainThread:1772506 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 00:05:53,483 DEBUG HandlerThread:1772506 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 00:05:53,483 INFO HandlerThread:1772506 [handler.py:finish():882] shutting down handler +2025-01-01 00:05:54,427 INFO WriterThread:1772506 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/run-5r8mx4bu.wandb +2025-01-01 00:05:54,482 INFO SenderThread:1772506 [sender.py:finish():1615] shutting down sender +2025-01-01 00:05:54,482 INFO SenderThread:1772506 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 00:05:54,482 INFO SenderThread:1772506 [file_pusher.py:join():175] waiting for file pusher diff --git a/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/logs/debug.log b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..960da7ce724403a5583b366b95c424bed7459cd9 --- /dev/null +++ b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_setup.py:_flush():76] Configure stats pid to 1772311 +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/settings +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/logs/debug.log +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/logs/debug-internal.log +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_init.py:init():569] calling init triggers +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu'} +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_init.py:init():619] starting backend +2024-12-31 23:59:53,322 INFO MainThread:1772311 [wandb_init.py:init():623] setting up manager +2024-12-31 23:59:53,327 INFO MainThread:1772311 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 23:59:53,328 INFO MainThread:1772311 [wandb_init.py:init():631] backend started and connected +2024-12-31 23:59:53,341 INFO MainThread:1772311 [wandb_init.py:init():720] updated telemetry +2024-12-31 23:59:53,351 INFO MainThread:1772311 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 23:59:53,651 INFO MainThread:1772311 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 23:59:53,687 INFO MainThread:1772311 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 23:59:53,687 INFO MainThread:1772311 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 23:59:54,062 INFO MainThread:1772311 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 23:59:54,062 INFO MainThread:1772311 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 23:59:54,063 INFO MainThread:1772311 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 23:59:54,063 INFO MainThread:1772311 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 23:59:54,064 INFO MainThread:1772311 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 23:59:54,064 INFO MainThread:1772311 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/5r8mx4bu/tb, True +2024-12-31 23:59:58,426 INFO MainThread:1772311 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-01 00:05:54,484 WARNING MsgRouterThr:1772311 [router.py:message_loop():77] message_loop has been closed diff --git a/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/run-5r8mx4bu.wandb b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/run-5r8mx4bu.wandb new file mode 100644 index 0000000000000000000000000000000000000000..010fbf0a73a8fbf515c98cccd7992bad93bd8889 Binary files /dev/null and b/5r8mx4bu/wandb/run-20241231_235953-5r8mx4bu/run-5r8mx4bu.wandb differ diff --git a/7j9cp4xk/episode_rosbags/episode_0_2025-01-01-00-07-19.bag b/7j9cp4xk/episode_rosbags/episode_0_2025-01-01-00-07-19.bag new file mode 100644 index 0000000000000000000000000000000000000000..f7b378c474d1e721ca5e3d604d15fab8c479586a --- /dev/null +++ b/7j9cp4xk/episode_rosbags/episode_0_2025-01-01-00-07-19.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3dc3cada073bb82494e91bfbc932e0d9c615d944961a96632417e88e7b1e084 +size 2192362400 diff --git a/8qih81be/.hydra/config.yaml b/8qih81be/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05485f98b3c08bece875a460fe25a0cce7504f90 --- /dev/null +++ b/8qih81be/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1006_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/8qih81be/.hydra/hydra.yaml b/8qih81be/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa3c3b43a98bc89c632df61fc965939a9e4b6320 --- /dev/null +++ b/8qih81be/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/8qih81be/.hydra/overrides.yaml b/8qih81be/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/8qih81be/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/8qih81be/episode_rosbags/aligned_depth_to_color_K.npy b/8qih81be/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/8qih81be/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/8qih81be/episode_rosbags/cam_tf_world.npy b/8qih81be/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/8qih81be/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/8qih81be/episode_rosbags/color_K.npy b/8qih81be/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/8qih81be/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/8qih81be/episode_rosbags/depth_K.npy b/8qih81be/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/8qih81be/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/8qih81be/eval_robot.log b/8qih81be/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..c846f58163c39f922846185de74087cbc3b631f1 --- /dev/null +++ b/8qih81be/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-01 11:22:09,102][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 11:22:09,105][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/8qih81be/eval_video/0_eval.mp4 b/8qih81be/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2a2f6cffcfcda91cca91cae0f7c0ffa899be1bcf --- /dev/null +++ b/8qih81be/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:080cfd687bc0e62059604a41877ba2852fdd3ff33430feaa80a1734c69cd3afa +size 1424925 diff --git a/8qih81be/eval_video/1_eval.mp4 b/8qih81be/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fcde4b06392936ef62ac65c19da65f1f81e19407 --- /dev/null +++ b/8qih81be/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d616a9f6d337515bde74ee45586fcef84b8963461c20fd5188603457e2718948 +size 861223 diff --git a/8qih81be/eval_video/2_eval.mp4 b/8qih81be/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f90865a845212ef8438c43f1fb4ddae8798e47c4 --- /dev/null +++ b/8qih81be/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52a5d239f2ed3d88df0e6886c31223d9873402737dca08bf5dd707b755bdac14 +size 968706 diff --git a/8qih81be/eval_video/3_eval.mp4 b/8qih81be/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..19a80504f6ee339612893347e239aaec5f91afef --- /dev/null +++ b/8qih81be/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6b478b7570f0157dada643a668466cedafb163e728f609dbac9303cc6318c3f +size 833329 diff --git a/8qih81be/eval_video/4_eval.mp4 b/8qih81be/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..365dc42157327cc9fdffd8cb4a50e7475fd0f677 --- /dev/null +++ b/8qih81be/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9e904a3822ce75d6da0fe98ab3e48ece0f8a14a13be2ca078c3de31fe17ab08 +size 996419 diff --git a/8qih81be/tb/events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 b/8qih81be/tb/events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 new file mode 100644 index 0000000000000000000000000000000000000000..e1f2e178577b431ce383b071f6969fa4d8e2e995 --- /dev/null +++ b/8qih81be/tb/events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f45080887ed16b1b6bd32c0e13fc0eee657187a0618fab7df0303781084281fb +size 6539 diff --git a/8qih81be/wandb/debug-internal.log b/8qih81be/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..17a32ab10d229fc155f655fef91c9e1c0ebf07b4 --- /dev/null +++ b/8qih81be/wandb/debug-internal.log @@ -0,0 +1,906 @@ +2025-01-01 11:22:14,855 INFO StreamThr :1793014 [internal.py:wandb_internal():85] W&B internal server running at pid: 1793014, started at: 2025-01-01 11:22:14.854579 +2025-01-01 11:22:14,857 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status +2025-01-01 11:22:14,868 INFO WriterThread:1793014 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/run-8qih81be.wandb +2025-01-01 11:22:14,868 DEBUG SenderThread:1793014 [sender.py:send():379] send: header +2025-01-01 11:22:14,876 DEBUG SenderThread:1793014 [sender.py:send():379] send: run +2025-01-01 11:22:15,125 INFO SenderThread:1793014 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files +2025-01-01 11:22:15,125 INFO SenderThread:1793014 [sender.py:_start_run_threads():1188] run started: 8qih81be with start time 1735748534.855132 +2025-01-01 11:22:15,137 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 11:22:15,137 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: check_version +2025-01-01 11:22:15,192 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 11:22:15,197 DEBUG HandlerThread:1793014 [system_info.py:__init__():26] System info init +2025-01-01 11:22:15,197 DEBUG HandlerThread:1793014 [system_info.py:__init__():41] System info init done +2025-01-01 11:22:15,197 INFO HandlerThread:1793014 [system_monitor.py:start():194] Starting system monitor +2025-01-01 11:22:15,197 INFO SystemMonitor:1793014 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 11:22:15,197 INFO HandlerThread:1793014 [system_monitor.py:probe():214] Collecting system info +2025-01-01 11:22:15,198 INFO SystemMonitor:1793014 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 11:22:15,198 INFO SystemMonitor:1793014 [interfaces.py:start():188] Started disk monitoring +2025-01-01 11:22:15,199 INFO SystemMonitor:1793014 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 11:22:15,199 INFO SystemMonitor:1793014 [interfaces.py:start():188] Started memory monitoring +2025-01-01 11:22:15,199 INFO SystemMonitor:1793014 [interfaces.py:start():188] Started network monitoring +2025-01-01 11:22:15,209 DEBUG HandlerThread:1793014 [system_info.py:probe():152] Probing system +2025-01-01 11:22:15,211 DEBUG HandlerThread:1793014 [system_info.py:_probe_git():137] Probing git +2025-01-01 11:22:15,216 DEBUG HandlerThread:1793014 [system_info.py:_probe_git():145] Probing git done +2025-01-01 11:22:15,216 DEBUG HandlerThread:1793014 [system_info.py:probe():200] Probing system done +2025-01-01 11:22:15,216 DEBUG HandlerThread:1793014 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T16:22:15.209885', 'startedAt': '2025-01-01T16:22:14.843448', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.949625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.01, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 696.8989372253418}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 11:22:15,216 INFO HandlerThread:1793014 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 11:22:15,216 INFO HandlerThread:1793014 [system_monitor.py:probe():227] Publishing system info +2025-01-01 11:22:15,216 DEBUG HandlerThread:1793014 [system_info.py:_save_code():44] Saving code +2025-01-01 11:22:15,222 DEBUG HandlerThread:1793014 [system_info.py:_save_code():67] Saving code done +2025-01-01 11:22:15,222 DEBUG HandlerThread:1793014 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 11:22:15,283 DEBUG HandlerThread:1793014 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 11:22:15,284 INFO HandlerThread:1793014 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 11:22:15,287 DEBUG SenderThread:1793014 [sender.py:send():379] send: files +2025-01-01 11:22:15,287 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 11:22:15,288 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 11:22:15,288 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 11:22:15,528 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 11:22:15,528 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: python_packages +2025-01-01 11:22:15,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:15,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:22:15,530 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:22:15,560 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/mh52qoln-diff.patch +2025-01-01 11:22:15,573 INFO HandlerThread:1793014 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/tb" + save: true +} +control { + relay_id: "f413d9117f934994a9f054bb1ef32783" +} +_info { + stream_id: "8qih81be" +} + +2025-01-01 11:22:15,573 DEBUG HandlerThread:1793014 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 11:22:15,581 INFO wandb-upload_0:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/kz92ej5k-wandb-metadata.json +2025-01-01 11:22:15,591 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/11xpddmv-code/FISH/eval_robot.py +2025-01-01 11:22:15,619 DEBUG SenderThread:1793014 [sender.py:send():379] send: telemetry +2025-01-01 11:22:15,624 DEBUG SenderThread:1793014 [sender.py:send():379] send: tbrecord +2025-01-01 11:22:15,625 DEBUG SenderThread:1793014 [sender.py:send():379] send: files +2025-01-01 11:22:15,625 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 with policy live +2025-01-01 11:22:15,810 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/svz00vyo-events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:22:16,127 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-metadata.json +2025-01-01 11:22:16,127 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/requirements.txt +2025-01-01 11:22:16,127 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/diff.patch +2025-01-01 11:22:16,128 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:22:16,128 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:16,128 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/code/FISH/eval_robot.py +2025-01-01 11:22:16,128 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/code/FISH +2025-01-01 11:22:16,128 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/code +2025-01-01 11:22:16,528 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:17,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:18,127 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:18,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:19,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:20,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:20,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:20,803 DEBUG SenderThread:1793014 [sender.py:send():379] send: config +2025-01-01 11:22:21,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:22,128 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:22,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:23,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:24,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:25,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:25,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:26,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:27,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:28,129 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:28,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:29,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:30,528 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:22:30,529 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:22:30,569 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:30,660 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:31,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:32,130 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:32,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:33,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:34,131 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:34,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:35,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:36,132 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:36,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:36,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:37,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:38,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:39,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:40,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:41,532 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:42,134 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:42,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:42,532 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:43,532 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:44,532 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:45,528 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:22:45,529 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:22:45,532 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:46,135 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:46,532 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:47,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:47,789 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:48,136 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:48,136 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/config.yaml +2025-01-01 11:22:48,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:49,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:50,137 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:50,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:51,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:52,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:53,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:53,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:54,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:55,534 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:56,138 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:56,534 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:57,534 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:58,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:58,534 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:59,534 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:00,139 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:00,528 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:23:00,529 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:23:00,569 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:01,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:02,534 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:03,534 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:03,619 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:04,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:05,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:06,141 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:06,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:07,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:08,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:08,843 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:09,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:10,142 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:10,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:11,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:12,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:13,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:14,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:14,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:15,200 DEBUG SystemMonitor:1793014 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 11:23:15,200 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:23:15,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:23:15,529 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:23:15,569 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:16,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:17,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:18,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:19,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:20,005 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:20,145 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:20,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:21,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:22,145 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:22,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:23,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:24,146 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:24,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:25,209 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:25,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:26,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:27,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:28,147 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:28,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:29,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:30,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:23:30,529 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:23:30,577 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:30,654 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:31,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:32,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:33,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:34,149 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:34,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:35,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:36,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:36,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:37,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:38,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:39,539 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:40,539 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:41,539 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:41,912 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:42,122 DEBUG SenderThread:1793014 [sender.py:send():379] send: files +2025-01-01 11:23:42,122 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_080cfd687bc0e6205960.mp4 with policy now +2025-01-01 11:23:42,122 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:23:42,123 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:23:42,123 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:23:42,124 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:23:42,152 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:42,152 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:23:42,152 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/0_eval_0_080cfd687bc0e6205960.mp4 +2025-01-01 11:23:42,152 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval +2025-01-01 11:23:42,152 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos +2025-01-01 11:23:42,152 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media +2025-01-01 11:23:42,350 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/608zbu6z-media/videos/eval/0_eval_0_080cfd687bc0e6205960.mp4 +2025-01-01 11:23:42,539 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:42,648 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/s5bvtm0i-media/videos/eval/0_eval_0_080cfd687bc0e6205960.mp4 +2025-01-01 11:23:43,539 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:44,152 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:44,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:45,201 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:23:45,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:23:45,530 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:23:45,573 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:46,540 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:47,133 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:47,541 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:48,153 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:48,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:49,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:50,541 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:51,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:52,135 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:52,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:53,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:54,155 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:54,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:55,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:56,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:57,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:57,937 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:58,157 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:58,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:59,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:00,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:24:00,530 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:24:00,573 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:01,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:02,543 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:03,158 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:24:03,399 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/0gi7em4o-events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:24:03,543 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:03,687 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:04,158 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:04,543 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:05,543 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:06,543 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:07,543 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:08,544 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:08,693 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:09,544 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:10,160 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:10,544 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:11,544 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:12,544 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:13,544 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:13,753 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:14,162 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:14,545 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:15,201 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:24:15,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:24:15,531 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:24:15,573 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:16,163 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:16,545 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:17,545 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:18,163 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:18,545 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:19,546 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:19,716 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:20,545 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:21,545 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:22,546 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:23,546 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:24,549 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:25,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:25,546 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:26,546 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:27,546 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:28,547 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:29,547 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:30,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:24:30,530 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:24:30,577 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:30,657 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:31,547 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:32,547 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:33,548 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:34,547 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:35,547 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:35,661 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:36,549 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:36,705 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:24:36,705 DEBUG SenderThread:1793014 [sender.py:send():379] send: files +2025-01-01 11:24:36,706 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_d616a9f6d337515bde74.mp4 with policy now +2025-01-01 11:24:36,706 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:24:36,706 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:24:36,706 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:24:36,915 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/6t89wdll-media/videos/eval/1_eval_1_d616a9f6d337515bde74.mp4 +2025-01-01 11:24:37,169 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:24:37,170 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/1_eval_1_d616a9f6d337515bde74.mp4 +2025-01-01 11:24:37,170 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval +2025-01-01 11:24:37,433 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/srhlcvks-media/videos/eval/1_eval_1_d616a9f6d337515bde74.mp4 +2025-01-01 11:24:37,550 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:38,170 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:38,551 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:39,556 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:40,559 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:41,539 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:41,558 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:42,554 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:43,556 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:44,172 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:44,556 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:45,202 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:24:45,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:24:45,535 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:24:45,577 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:46,558 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:46,648 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:47,557 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:48,173 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:48,568 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:49,558 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:50,174 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:50,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:51,559 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:52,174 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:52,559 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:52,637 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:53,559 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:54,560 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:55,559 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:56,560 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:57,560 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:58,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:58,560 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:59,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:00,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:25:00,532 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:25:00,573 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:01,560 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:02,561 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:03,561 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:03,589 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:04,561 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:05,047 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:25:05,047 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:25:05,047 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:25:05,179 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:25:05,179 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:25:05,411 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/tw9zmb98-events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:25:05,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:06,179 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:06,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:07,561 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:08,561 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:09,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:09,564 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:10,561 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:11,561 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:12,181 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:12,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:13,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:14,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:14,890 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:15,202 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:25:15,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:25:15,531 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:25:15,563 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:16,183 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:16,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:17,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:18,183 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:18,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:19,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:20,184 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:20,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:20,563 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:21,563 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:22,563 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:23,563 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:24,564 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:25,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:25,564 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:26,564 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:27,564 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:28,187 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:28,564 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:29,564 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:30,188 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:30,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:25:30,532 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:25:30,573 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:30,690 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:31,564 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:32,565 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:33,565 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:34,565 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:35,565 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:36,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:36,565 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:37,565 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:38,565 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:39,565 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:40,565 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:41,574 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:41,752 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:41,894 DEBUG SenderThread:1793014 [sender.py:send():379] send: files +2025-01-01 11:25:41,894 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_52a5d239f2ed3d88df0e.mp4 with policy now +2025-01-01 11:25:41,901 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:25:41,901 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:25:41,901 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:25:41,901 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:25:42,104 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/uu5x662e-media/videos/eval/2_eval_2_52a5d239f2ed3d88df0e.mp4 +2025-01-01 11:25:42,192 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:25:42,192 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:42,192 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/2_eval_2_52a5d239f2ed3d88df0e.mp4 +2025-01-01 11:25:42,192 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval +2025-01-01 11:25:42,454 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/1bz77e3s-media/videos/eval/2_eval_2_52a5d239f2ed3d88df0e.mp4 +2025-01-01 11:25:42,570 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:43,569 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:44,192 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:44,571 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:45,203 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:25:45,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:25:45,536 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:25:45,577 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:46,573 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:46,937 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:47,576 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:48,194 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:48,581 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:49,579 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:50,575 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:51,575 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:52,039 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:52,576 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:53,579 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:54,196 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:54,580 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:55,580 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:56,583 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:57,580 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:57,998 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:58,197 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:58,579 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:59,584 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:00,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:26:00,536 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:26:00,582 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:00,895 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:26:00,895 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:26:00,896 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:26:01,198 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:26:01,198 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:26:01,589 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:02,198 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:02,592 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:03,590 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:03,896 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:04,589 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:05,592 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:06,594 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:07,594 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:08,200 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:08,593 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:09,595 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:09,853 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:10,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:11,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:12,202 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:12,598 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:13,598 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:14,202 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:14,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:14,923 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:15,203 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:26:15,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:26:15,534 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:26:15,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:16,204 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:16,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:17,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:18,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:19,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:20,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:20,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:21,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:22,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:23,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:24,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:25,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:26,541 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:26,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:27,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:28,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:29,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:30,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:26:30,534 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:26:30,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:31,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:31,961 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:32,209 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:32,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:33,604 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:34,210 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:34,305 DEBUG SenderThread:1793014 [sender.py:send():379] send: files +2025-01-01 11:26:34,305 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c6b478b7570f0157dada.mp4 with policy now +2025-01-01 11:26:34,307 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:26:34,308 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:26:34,308 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:26:34,308 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:26:34,508 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/eeubojar-media/videos/eval/3_eval_3_c6b478b7570f0157dada.mp4 +2025-01-01 11:26:34,599 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:35,210 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:26:35,210 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/3_eval_3_c6b478b7570f0157dada.mp4 +2025-01-01 11:26:35,211 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval +2025-01-01 11:26:35,472 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/dmli1tfo-media/videos/eval/3_eval_3_c6b478b7570f0157dada.mp4 +2025-01-01 11:26:35,604 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:36,211 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:36,600 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:37,550 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:37,603 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:38,606 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:39,609 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:40,212 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:40,606 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:41,608 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:42,607 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:43,548 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:43,624 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:44,635 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:45,204 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:26:45,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:26:45,536 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:26:45,630 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:46,214 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:46,622 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:47,622 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:48,215 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:48,617 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:49,245 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:49,622 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:50,215 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:50,622 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:51,626 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:52,627 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:53,628 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:54,549 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:54,629 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:55,631 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:56,631 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:57,634 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:57,960 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:26:57,961 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:26:57,962 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:26:58,218 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:26:58,218 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:26:58,503 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/jv7vsvf1-events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:26:58,637 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:59,632 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:59,962 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:00,219 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:00,547 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:27:00,547 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:27:00,643 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:01,637 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:02,645 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:03,638 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:04,220 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:04,636 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:05,549 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:05,644 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:06,642 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:07,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:08,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:09,642 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:10,222 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:10,552 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:10,649 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:11,647 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:12,645 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:13,645 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:14,224 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:14,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:15,204 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:27:15,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:27:15,537 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:27:15,634 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:15,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:16,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:17,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:18,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:19,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:20,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:21,551 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:21,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:22,647 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:23,647 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:24,647 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:25,647 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:26,551 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:26,647 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:27,647 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:28,647 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:29,648 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:30,229 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:30,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:27:30,537 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:27:30,648 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:31,625 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:31,648 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:32,648 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:33,648 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:34,648 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:35,554 DEBUG SenderThread:1793014 [sender.py:send():379] send: files +2025-01-01 11:27:35,555 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_b9e904a3822ce75d6da0.mp4 with policy now +2025-01-01 11:27:35,561 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:27:35,562 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:27:35,562 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:35,562 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:35,651 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:35,769 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/9c5dleaq-media/videos/eval/4_eval_4_b9e904a3822ce75d6da0.mp4 +2025-01-01 11:27:36,231 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:27:36,231 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:36,231 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/4_eval_4_b9e904a3822ce75d6da0.mp4 +2025-01-01 11:27:36,231 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval +2025-01-01 11:27:36,491 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/54r0jryy-media/videos/eval/4_eval_4_b9e904a3822ce75d6da0.mp4 +2025-01-01 11:27:36,656 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:37,610 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:37,664 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:38,232 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:38,658 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:39,662 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:40,659 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:41,657 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:42,233 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:42,657 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:42,658 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:43,661 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:44,667 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:45,204 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:27:45,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:27:45,542 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:27:45,665 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:46,666 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:47,663 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:47,765 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:48,235 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:48,664 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:49,673 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:50,674 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:51,667 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:52,236 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:52,674 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:53,561 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:53,677 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:54,667 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:55,673 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:56,675 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:57,675 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:57,762 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:27:57,762 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:27:57,959 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:27:58,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:27:58,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:58,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:27:58,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media +2025-01-01 11:27:58,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table/eval +2025-01-01 11:27:58,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table +2025-01-01 11:27:58,243 INFO wandb-upload_2:1793014 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp3itgor1_ +2025-01-01 11:27:58,394 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:27:58,402 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:27:58,453 INFO wandb-upload_0:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/al84qfgm-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:27:58,479 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/ta6u3jjt-events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:27:58,641 INFO SenderThread:1793014 [sender.py:send_request_log_artifact():1525] logged artifact run-8qih81be-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY4Nzg2NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MjQ4NQ==', 'latestArtifact': None}} +2025-01-01 11:27:58,642 DEBUG SenderThread:1793014 [sender.py:send():379] send: files +2025-01-01 11:27:58,642 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-01 11:27:58,642 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:27:58,643 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,643 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:58,644 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,644 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:27:58,644 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,648 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,648 DEBUG SenderThread:1793014 [sender.py:send():379] send: telemetry +2025-01-01 11:27:58,648 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,648 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,648 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,649 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,649 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,649 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,649 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,650 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,650 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,650 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,650 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,651 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,651 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,651 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,651 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,652 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,652 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,652 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,652 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,653 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,653 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,653 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,653 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:27:58,674 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:58,873 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/j0y7gcuh-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:27:59,000 INFO wandb-upload_1:1793014 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwi4_w24g +2025-01-01 11:27:59,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:27:59,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:27:59,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-01 11:27:59,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table/eval +2025-01-01 11:27:59,402 INFO SenderThread:1793014 [sender.py:send_request_log_artifact():1525] logged artifact run-8qih81be-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY4Nzg3Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MjQ4Nw==', 'latestArtifact': None}} +2025-01-01 11:27:59,402 DEBUG SenderThread:1793014 [sender.py:send():379] send: files +2025-01-01 11:27:59,402 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json with policy now +2025-01-01 11:27:59,402 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:27:59,403 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:59,403 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:59,574 INFO wandb-upload_0:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/tlk04g5i-media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-01 11:27:59,674 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:59,825 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/whp3g0j5-media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-01 11:28:00,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:28:00,240 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:28:00,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:28:00,538 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:28:00,674 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:01,674 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:02,675 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:03,675 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:04,638 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:04,675 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:05,689 DEBUG SenderThread:1793014 [sender.py:send():379] send: exit +2025-01-01 11:28:05,689 INFO SenderThread:1793014 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 11:28:05,690 INFO SenderThread:1793014 [sender.py:send_exit():588] handling runtime: 350 +2025-01-01 11:28:05,690 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:28:05,690 INFO SenderThread:1793014 [sender.py:send_exit():594] send defer +2025-01-01 11:28:05,690 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:05,690 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 11:28:05,690 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:05,690 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 11:28:05,690 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 1 +2025-01-01 11:28:05,690 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:05,691 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 11:28:05,691 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:05,691 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 11:28:05,691 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 2 +2025-01-01 11:28:05,691 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:05,691 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 11:28:05,691 INFO HandlerThread:1793014 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 11:28:05,691 DEBUG SystemMonitor:1793014 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 11:28:05,691 DEBUG SystemMonitor:1793014 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 11:28:05,692 INFO HandlerThread:1793014 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 11:28:05,692 INFO HandlerThread:1793014 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 11:28:05,743 INFO HandlerThread:1793014 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 11:28:05,743 INFO HandlerThread:1793014 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 11:28:05,744 INFO HandlerThread:1793014 [interfaces.py:finish():200] Joined network monitor +2025-01-01 11:28:05,744 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:05,744 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 11:28:05,744 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 3 +2025-01-01 11:28:05,744 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:28:05,744 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:05,744 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 11:28:05,745 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:05,745 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 11:28:05,745 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 4 +2025-01-01 11:28:05,745 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:05,745 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 11:28:06,242 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:28:08,243 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:28:12,037 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:28:12,037 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:12,037 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 11:28:12,038 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:12,038 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 11:28:12,039 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 5 +2025-01-01 11:28:12,039 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:28:12,039 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:28:12,039 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:28:12,040 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:12,040 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 11:28:12,040 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:28:12,041 DEBUG SenderThread:1793014 [sender.py:send():379] send: summary +2025-01-01 11:28:12,044 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:28:12,045 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:12,045 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 11:28:12,045 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 6 +2025-01-01 11:28:12,045 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:12,045 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 11:28:12,045 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:12,045 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 11:28:12,049 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:12,171 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 7 +2025-01-01 11:28:12,171 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:12,171 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 11:28:12,171 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:12,171 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 11:28:12,244 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:28:12,245 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/config.yaml +2025-01-01 11:28:12,690 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:28:15,879 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 8 +2025-01-01 11:28:15,880 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:28:15,880 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:15,880 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 11:28:15,880 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:15,880 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 11:28:15,880 INFO SenderThread:1793014 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 11:28:15,883 INFO SenderThread:1793014 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 11:28:15,884 INFO SenderThread:1793014 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 11:28:15,898 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 9 +2025-01-01 11:28:15,898 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:15,898 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 11:28:15,898 DEBUG SenderThread:1793014 [sender.py:send():379] send: artifact +2025-01-01 11:28:16,151 INFO SenderThread:1793014 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjk2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 11:28:16,151 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:16,151 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 11:28:16,151 INFO SenderThread:1793014 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 11:28:16,245 INFO SenderThread:1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:28:16,246 INFO SenderThread:1793014 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files +2025-01-01 11:28:16,246 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/config.yaml config.yaml +2025-01-01 11:28:16,246 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json wandb-summary.json +2025-01-01 11:28:16,246 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/requirements.txt requirements.txt +2025-01-01 11:28:16,246 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-metadata.json wandb-metadata.json +2025-01-01 11:28:16,246 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:28:16,249 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log output.log +2025-01-01 11:28:16,249 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/diff.patch diff.patch +2025-01-01 11:28:16,251 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 11:28:16,251 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/1_eval_1_d616a9f6d337515bde74.mp4 media/videos/eval/1_eval_1_d616a9f6d337515bde74.mp4 +2025-01-01 11:28:16,251 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/4_eval_4_b9e904a3822ce75d6da0.mp4 media/videos/eval/4_eval_4_b9e904a3822ce75d6da0.mp4 +2025-01-01 11:28:16,251 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/3_eval_3_c6b478b7570f0157dada.mp4 media/videos/eval/3_eval_3_c6b478b7570f0157dada.mp4 +2025-01-01 11:28:16,251 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/2_eval_2_52a5d239f2ed3d88df0e.mp4 media/videos/eval/2_eval_2_52a5d239f2ed3d88df0e.mp4 +2025-01-01 11:28:16,252 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/0_eval_0_080cfd687bc0e6205960.mp4 media/videos/eval/0_eval_0_080cfd687bc0e6205960.mp4 +2025-01-01 11:28:16,253 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-01 11:28:16,253 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:28:16,253 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 10 +2025-01-01 11:28:16,253 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:16,253 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 11:28:16,254 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:16,254 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 11:28:16,254 INFO SenderThread:1793014 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:28:16,436 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/requirements.txt +2025-01-01 11:28:16,441 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:28:16,462 INFO wandb-upload_0:1793014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/config.yaml +2025-01-01 11:28:16,491 INFO wandb-upload_4:1793014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:28:16,548 INFO wandb-upload_3:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/nbsw5bpi-events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:28:16,692 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:28:16,692 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:28:16,748 INFO Thread-11 (_thread_body):1793014 [sender.py:transition_state():614] send defer: 11 +2025-01-01 11:28:16,748 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:16,748 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 11:28:16,749 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:16,749 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 11:28:16,749 INFO SenderThread:1793014 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 11:28:16,750 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 12 +2025-01-01 11:28:16,750 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:16,750 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 11:28:16,750 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:16,750 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 11:28:16,750 INFO SenderThread:1793014 [file_stream.py:finish():601] file stream finish called +2025-01-01 11:28:16,968 INFO SenderThread:1793014 [file_stream.py:finish():605] file stream finish is done +2025-01-01 11:28:16,968 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 13 +2025-01-01 11:28:16,968 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:16,968 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 11:28:16,968 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:16,968 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 11:28:16,968 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 14 +2025-01-01 11:28:16,968 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:16,968 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 11:28:16,969 DEBUG SenderThread:1793014 [sender.py:send():379] send: final +2025-01-01 11:28:16,969 DEBUG SenderThread:1793014 [sender.py:send():379] send: footer +2025-01-01 11:28:16,969 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:16,969 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 11:28:16,969 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:28:16,969 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:28:16,970 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 11:28:16,970 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:28:16,970 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 11:28:16,970 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: server_info +2025-01-01 11:28:16,971 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 11:28:16,973 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:17,032 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:28:17,033 INFO MainThread:1793014 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 11:28:17,033 INFO MainThread:1793014 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 11:28:17,033 INFO MainThread:1793014 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 11:28:17,034 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 11:28:17,034 INFO HandlerThread:1793014 [handler.py:finish():882] shutting down handler +2025-01-01 11:28:17,970 INFO WriterThread:1793014 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/run-8qih81be.wandb +2025-01-01 11:28:18,033 INFO SenderThread:1793014 [sender.py:finish():1615] shutting down sender +2025-01-01 11:28:18,033 INFO SenderThread:1793014 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:28:18,033 INFO SenderThread:1793014 [file_pusher.py:join():175] waiting for file pusher diff --git a/8qih81be/wandb/debug.log b/8qih81be/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e0140b5afaeabfea4e7ead39a34f7c03eb2b1f7b --- /dev/null +++ b/8qih81be/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_setup.py:_flush():76] Configure stats pid to 1792840 +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/settings +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/logs/debug.log +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/logs/debug-internal.log +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_init.py:init():569] calling init triggers +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be'} +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_init.py:init():619] starting backend +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_init.py:init():623] setting up manager +2025-01-01 11:22:14,853 INFO MainThread:1792840 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 11:22:14,854 INFO MainThread:1792840 [wandb_init.py:init():631] backend started and connected +2025-01-01 11:22:14,868 INFO MainThread:1792840 [wandb_init.py:init():720] updated telemetry +2025-01-01 11:22:14,874 INFO MainThread:1792840 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 11:22:15,136 INFO MainThread:1792840 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 11:22:15,186 INFO MainThread:1792840 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 11:22:15,186 INFO MainThread:1792840 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 11:22:15,528 INFO MainThread:1792840 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 11:22:15,529 INFO MainThread:1792840 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 11:22:15,529 INFO MainThread:1792840 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 11:22:15,529 INFO MainThread:1792840 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 11:22:15,531 INFO MainThread:1792840 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 11:22:15,531 INFO MainThread:1792840 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/tb, True +2025-01-01 11:22:20,803 INFO MainThread:1792840 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-01 11:28:18,034 WARNING MsgRouterThr:1792840 [router.py:message_loop():77] message_loop has been closed diff --git a/8qih81be/wandb/run-20250101_112214-8qih81be/files/code/FISH/eval_robot.py b/8qih81be/wandb/run-20250101_112214-8qih81be/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/8qih81be/wandb/run-20250101_112214-8qih81be/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/8qih81be/wandb/run-20250101_112214-8qih81be/files/config.yaml b/8qih81be/wandb/run-20250101_112214-8qih81be/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab544fda17a4841970da921e65a8ab7a75f460be --- /dev/null +++ b/8qih81be/wandb/run-20250101_112214-8qih81be/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1006_0' +wandb_notes: + desc: null + value: 1006_0_req_1352_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735748534 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/8qih81be/wandb/run-20250101_112214-8qih81be/files/diff.patch b/8qih81be/wandb/run-20250101_112214-8qih81be/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..625cf877496560323ca22cd6b0a075ba511a2dc6 --- /dev/null +++ b/8qih81be/wandb/run-20250101_112214-8qih81be/files/diff.patch @@ -0,0 +1,106 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..01bdd86 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1007_0' ++# wandb_run_id: '1002_0' + ++# all books, 6/20 demos per book ++# RGBD+mask+act history ++# wandb_run_id: '1007_0' ++ ++# RGBD+mask+contact(all ftrs)+act history ++wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..50493fe 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -490,7 +490,8 @@ class Workspace: + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] +- total_num_episodes += filtered_run.summary_metrics['episode'] ++ # total_num_episodes += filtered_run.summary_metrics['episode'] ++ total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes +@@ -513,7 +514,6 @@ class Workspace: + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + +- + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index e3031af..6f97e26 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,11 +330,11 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.01 +- self.move_to_rot_tolerance = 2.0 ++ self.move_to_trans_tolerance = 0.005 ++ self.move_to_rot_tolerance = 1.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) +- self.move_to_target_timeout = 15.0 ++ self.move_to_target_timeout = 5.0 + self.move_to_target_max_rotation = 25.0 + # only can use one of these at a time + # assert use_roll + use_pitch + use_yaw <= 1, "Can only use one of roll, pitch, yaw" diff --git a/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json b/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json new file mode 100644 index 0000000000000000000000000000000000000000..6bdff6fffa9b038190acdbc6f7b116097984799e --- /dev/null +++ b/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[25, 50, 0.5, 0.33669589463162186, 0.6620035767231229, 0.3672531694439306, 0.6438583284328754, 0.387978680454247, 0.6119417696967355, 0.4043694697511723, 0.5970221161648301]]} \ No newline at end of file diff --git a/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/0_eval_0_080cfd687bc0e6205960.mp4 b/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/0_eval_0_080cfd687bc0e6205960.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2a2f6cffcfcda91cca91cae0f7c0ffa899be1bcf --- /dev/null +++ b/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/0_eval_0_080cfd687bc0e6205960.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:080cfd687bc0e62059604a41877ba2852fdd3ff33430feaa80a1734c69cd3afa +size 1424925 diff --git a/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/1_eval_1_d616a9f6d337515bde74.mp4 b/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/1_eval_1_d616a9f6d337515bde74.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fcde4b06392936ef62ac65c19da65f1f81e19407 --- /dev/null +++ b/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/1_eval_1_d616a9f6d337515bde74.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d616a9f6d337515bde74ee45586fcef84b8963461c20fd5188603457e2718948 +size 861223 diff --git a/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/2_eval_2_52a5d239f2ed3d88df0e.mp4 b/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/2_eval_2_52a5d239f2ed3d88df0e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f90865a845212ef8438c43f1fb4ddae8798e47c4 --- /dev/null +++ b/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/2_eval_2_52a5d239f2ed3d88df0e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52a5d239f2ed3d88df0e6886c31223d9873402737dca08bf5dd707b755bdac14 +size 968706 diff --git a/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/3_eval_3_c6b478b7570f0157dada.mp4 b/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/3_eval_3_c6b478b7570f0157dada.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..19a80504f6ee339612893347e239aaec5f91afef --- /dev/null +++ b/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/3_eval_3_c6b478b7570f0157dada.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6b478b7570f0157dada643a668466cedafb163e728f609dbac9303cc6318c3f +size 833329 diff --git a/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/4_eval_4_b9e904a3822ce75d6da0.mp4 b/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/4_eval_4_b9e904a3822ce75d6da0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..365dc42157327cc9fdffd8cb4a50e7475fd0f677 --- /dev/null +++ b/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/4_eval_4_b9e904a3822ce75d6da0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9e904a3822ce75d6da0fe98ab3e48ece0f8a14a13be2ca078c3de31fe17ab08 +size 996419 diff --git a/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log b/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..bdce81c494c4143740f56a8dfe3288becd5ef2ab --- /dev/null +++ b/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log @@ -0,0 +1,307 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735748540.804010]: resetting environment +[INFO] [1735748540.812605]: cleared current plan +[INFO] [1735748540.813764]: moving to home +Move to target timeout: 5.000646352767944. Trans error: 0.006063592620193958. Rot error: 0.6319731023777525 +Move to target timeout: 5.0997703075408936. Trans error: 0.006251493226840755. Rot error: 1.4802715549310272 +[INFO] [1735748551.916900]: reached home +[INFO] [1735748551.918247]: reset action history +[INFO] [1735748554.628086]: environment reset +Starting episode 0 +[INFO] [1735748554.633441]: resetting environment +[INFO] [1735748554.639375]: cleared current plan +[INFO] [1735748554.640401]: moving to home +Move to target timeout: 5.00058650970459. Trans error: 0.005796351470053196. Rot error: 0.5626831079295407 +Move to target timeout: 5.099817752838135. Trans error: 0.006354032886955458. Rot error: 1.5785863029415403 +[INFO] [1735748565.743663]: reached home +[INFO] [1735748565.744949]: reset action history +[INFO] [1735748568.455504]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735748573.731572]: resetting environment +[INFO] [1735748573.736224]: cleared current plan +[INFO] [1735748573.737126]: moving to home +Move to target timeout: 5.00063419342041. Trans error: 0.005766535643488169. Rot error: 0.5730935035515325 +Move to target timeout: 5.099754333496094. Trans error: 0.006401541742965536. Rot error: 1.5969371637675691 +[INFO] [1735748584.840130]: reached home +[INFO] [1735748584.841425]: reset action history +[INFO] [1735748587.551493]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/episode_rosbags/episode_0_2025-01-01-11-23-07.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1792840.log +started roslaunch server http://158.130.50.37:43137/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1793123] +process[print_text-2]: started with pid [1793147] +process[rosbag_record-3]: started with pid [1793148] +started bagging! +For topic gripper_width: timestamp difference is -101820427 for nearest: 1735748597778625053 - target: 1735748597880445480 at idx 134 +For topic gripper_width: timestamp difference is 100231347 for nearest: 1735748600545292792 - target: 1735748600445061445 at idx 164 +For topic gripper_width: timestamp difference is 102347039 for nearest: 1735748603145327233 - target: 1735748603042980194 at idx 204 +For topic gripper_width: timestamp difference is 104834501 for nearest: 1735748606278628055 - target: 1735748606173793554 at idx 236 +For topic gripper_width: timestamp difference is 105043095 for nearest: 1735748606545294445 - target: 1735748606440251350 at idx 238 +For topic gripper_width: timestamp difference is 108995775 for nearest: 1735748611445295433 - target: 1735748611336299658 at idx 249 +preempting episode +[INFO] [1735748621.134473]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1735748621.912412]: Stored episode 1. +[INFO] [1735748621.913369]: Saving video... +[INFO] [1735748622.106539]: Video saved! +Starting episode 1 +[INFO] [1735748622.123379]: resetting environment +[INFO] [1735748622.130704]: cleared current plan +[INFO] [1735748622.131597]: moving to home +Move to target timeout: 5.000701427459717. Trans error: 0.008989774622023106. Rot error: 1.3695691303190412 +Move to target timeout: 5.000543832778931. Trans error: 0.006200415461706595. Rot error: 1.0104160247020608 +[INFO] [1735748633.136496]: reached home +[INFO] [1735748633.136992]: reset action history +[INFO] [1735748635.895294]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735748642.646137]: resetting environment +[INFO] [1735748642.649888]: cleared current plan +[INFO] [1735748642.650117]: moving to home +Move to target timeout: 5.001338243484497. Trans error: 0.006050876807421446. Rot error: 0.6504737883120796 +Move to target timeout: 5.099688529968262. Trans error: 0.006250880961803502. Rot error: 1.4925745594694415 +[INFO] [1735748653.753039]: reached home +[INFO] [1735748653.753448]: reset action history +[INFO] [1735748656.461823]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/episode_rosbags/episode_1_2025-01-01-11-24-16.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1792840.log +started roslaunch server http://158.130.50.37:34511/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1793279] +process[print_text-5]: started with pid [1793280] +process[rosbag_record-6]: started with pid [1793281] +started bagging! +preempting episode +[INFO] [1735748675.633245]: Storing episode... +[rosbag_record-6] killing on exit[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735748676.394835]: Stored episode 2. +[INFO] [1735748676.395045]: Saving video... +[INFO] [1735748676.651756]: Video saved! +Starting episode 2 +[INFO] [1735748676.678666]: resetting environment +[INFO] [1735748676.705600]: cleared current plan +[INFO] [1735748676.707433]: moving to home +Move to target timeout: 5.0020318031311035. Trans error: 0.008842650800943375. Rot error: 1.3144809033375748 +Move to target timeout: 5.097257852554321. Trans error: 0.006250678016602967. Rot error: 1.0159099207092255 +[INFO] [1735748687.816522]: reached home +[INFO] [1735748687.818838]: reset action history +[INFO] [1735748690.594045]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735748704.767850]: resetting environment +[INFO] [1735748704.787806]: cleared current plan +[INFO] [1735748704.788015]: moving to home +Move to target timeout: 5.000982046127319. Trans error: 0.006069135386496782. Rot error: 0.6536614232391569 +Move to target timeout: 5.098495721817017. Trans error: 0.006332403213271913. Rot error: 1.501109354469627 +[INFO] [1735748715.891596]: reached home +[INFO] [1735748715.892056]: reset action history +[INFO] [1735748718.600296]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/episode_rosbags/episode_2_2025-01-01-11-25-18.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1792840.log +started roslaunch server http://158.130.50.37:40145/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1793421] +process[print_text-8]: started with pid [1793422] +process[rosbag_record-9]: started with pid [1793446] +started bagging! +For topic gripper_width: timestamp difference is 101281620 for nearest: 1735748725578632093 - target: 1735748725477350473 at idx 93 +For topic gripper_width: timestamp difference is 101483311 for nearest: 1735748725845294680 - target: 1735748725743811369 at idx 95 +For topic gripper_width: timestamp difference is 103732849 for nearest: 1735748728678653265 - target: 1735748728574920416 at idx 136 +preempting episode +[INFO] [1735748740.599086]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1735748741.324437]: Stored episode 3. +[INFO] [1735748741.328177]: Saving video... +[WARN] [1735748741.750393]: Plan exhausted +[WARN] [1735748741.790410]: Plan exhausted +[WARN] [1735748741.830986]: Plan exhausted +[WARN] [1735748741.870298]: Plan exhausted +[WARN] [1735748741.909934]: Plan exhausted +[INFO] [1735748741.855036]: Video saved! +Starting episode 3 +[INFO] [1735748741.908112]: resetting environment +[INFO] [1735748741.935705]: cleared current plan +[INFO] [1735748741.935957]: moving to home +Move to target timeout: 5.001161813735962. Trans error: 0.009125432930886745. Rot error: 1.422925053142918 +Move to target timeout: 5.099911689758301. Trans error: 0.006230721831392119. Rot error: 1.014547615561742 +[INFO] [1735748753.044026]: reached home +[INFO] [1735748753.050532]: reset action history +For topic color: timestamp difference is -133224249 for nearest: 1735748755752732754 - target: 1735748755885957003 at idx 29 +[INFO] [1735748755.976209]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735748760.760080]: resetting environment +[INFO] [1735748760.791381]: cleared current plan +[INFO] [1735748760.798618]: moving to home +Move to target timeout: 5.002576589584351. Trans error: 0.006124742794781923. Rot error: 0.6709264170301137 +Move to target timeout: 5.098871946334839. Trans error: 0.006322771533245207. Rot error: 1.5014965060256915 +[INFO] [1735748771.911310]: reached home +[INFO] [1735748771.914134]: reset action history +[INFO] [1735748774.695104]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/episode_rosbags/episode_3_2025-01-01-11-26-14.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1792840.log +started roslaunch server http://158.130.50.37:42621/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1793600] +process[print_text-11]: started with pid [1793601] +process[rosbag_record-12]: started with pid [1793602] +started bagging! +For topic gripper_width: timestamp difference is -140145760 for nearest: 1735748790678626079 - target: 1735748790818771839 at idx 210 +preempting episode +[INFO] [1735748793.242252]: Storing episode... +[print_text-11] killing on exit +[pub_text-10] killing on exit +[rosbag_record-12] killing on exit +[WARN] [1735748794.349736]: Plan exhausted +[INFO] [1735748794.016612]: Stored episode 4. +[INFO] [1735748794.024654]: Saving video... +[INFO] [1735748794.260883]: Video saved! +Starting episode 4 +[INFO] [1735748794.312194]: resetting environment +[INFO] [1735748794.361308]: cleared current plan +[INFO] [1735748794.363617]: moving to home +Move to target timeout: 5.0056750774383545. Trans error: 0.0072539192624390125. Rot error: 0.8836877635706815 +Move to target timeout: 5.002500534057617. Trans error: 0.006236067104700083. Rot error: 1.011811765759742 +[INFO] [1735748805.400656]: reached home +[INFO] [1735748805.407314]: reset action history +[INFO] [1735748808.199575]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735748817.752742]: resetting environment +[INFO] [1735748817.797014]: cleared current plan +[INFO] [1735748817.805488]: moving to home +Move to target timeout: 5.00229024887085. Trans error: 0.0060905530117452145. Rot error: 0.6625449090581153 +Move to target timeout: 5.09676456451416. Trans error: 0.0063448756774022955. Rot error: 1.5038019168041667 +[INFO] [1735748828.921987]: reached home +[INFO] [1735748828.924752]: reset action history +[INFO] [1735748831.651224]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/episode_rosbags/episode_4_2025-01-01-11-27-11.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1792840.log +started roslaunch server http://158.130.50.37:42735/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1793718] +process[print_text-14]: started with pid [1793719] +process[rosbag_record-15]: started with pid [1793743] +started bagging! +For topic gripper_width: timestamp difference is 100771828 for nearest: 1735748848745292349 - target: 1735748848644520521 at idx 231 +preempting episode +[INFO] [1735748854.466117]: Storing episode... +[print_text-14] killing on exit +[pub_text-13] killing on exit +[rosbag_record-15] killing on exit +[INFO] [1735748855.194358]: Stored episode 5. +[INFO] [1735748855.197954]: Saving video... +[INFO] [1735748855.509008]: Video saved! +[INFO] [1735748855.565636]: resetting environment +[INFO] [1735748855.609968]: cleared current plan +[INFO] [1735748855.610203]: moving to home +Move to target timeout: 5.002002954483032. Trans error: 0.008681077510118484. Rot error: 1.3411581405908268 +Move to target timeout: 5.095853805541992. Trans error: 0.0062414387645393435. Rot error: 1.0253591939577495 +[INFO] [1735748866.720953]: reached home +[INFO] [1735748866.728718]: reset action history +[INFO] [1735748869.522136]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/8qih81be/wandb/run-20250101_112214-8qih81be/files/requirements.txt b/8qih81be/wandb/run-20250101_112214-8qih81be/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/8qih81be/wandb/run-20250101_112214-8qih81be/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-metadata.json b/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d0a2d9b61f8db8a5d1d440150f0b09db138bffb8 --- /dev/null +++ b/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T16:22:15.209885", + "startedAt": "2025-01-01T16:22:14.843448", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.949625, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.01, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 696.8989372253418 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json b/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9b128b95c6efe03fe106f68f43bb2523833e35a4 --- /dev/null +++ b/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "080cfd687bc0e62059604a41877ba2852fdd3ff33430feaa80a1734c69cd3afa", "size": 1424925, "path": "media/videos/eval/0_eval_0_080cfd687bc0e6205960.mp4"}, "global_step": 439, "_timestamp": 1735748878.3035634, "_runtime": 343.4484312534332, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "d616a9f6d337515bde74ee45586fcef84b8963461c20fd5188603457e2718948", "size": 861223, "path": "media/videos/eval/1_eval_1_d616a9f6d337515bde74.mp4"}, "eval/num_success": 0.0, "episode": 50.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "52a5d239f2ed3d88df0e6886c31223d9873402737dca08bf5dd707b755bdac14", "size": 968706, "path": "media/videos/eval/2_eval_2_52a5d239f2ed3d88df0e.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "c6b478b7570f0157dada643a668466cedafb163e728f609dbac9303cc6318c3f", "size": 833329, "path": "media/videos/eval/3_eval_3_c6b478b7570f0157dada.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "b9e904a3822ce75d6da0fe98ab3e48ece0f8a14a13be2ca078c3de31fe17ab08", "size": 996419, "path": "media/videos/eval/4_eval_4_b9e904a3822ce75d6da0.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://rnzs09r5qod3tjpe4ug0d7kqpjeolvhtyoxlg80vti7e90x4vk0cqk32eef09quu2g0cghxxvjrria4bo50jam82isuput88wpdomhl0f52t280t8jurlytduxdx0xe2/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://pxzbpn98ualv9gcr63nz1bexgf4ttdirqcbbndh5y6y6fiwcupxfk9wprnhk9flstdr14pvdmaf6w49rbydf19uf0etljh6mrfqnqb2atm17h4h3ymq414anaed1gruw:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 25, "total_num_episodes": 50, "total_success_rate": 0.5, "total_success_rate_lb_0.01": 0.33669589463162186, "total_success_rate_ub_0.01": 0.6620035767231229, "total_success_rate_lb_0.025": 0.3672531694439306, "total_success_rate_ub_0.025": 0.6438583284328754, "total_success_rate_lb_0.05": 0.387978680454247, "total_success_rate_ub_0.05": 0.6119417696967355, "total_success_rate_lb_0.1": 0.4043694697511723, "total_success_rate_ub_0.1": 0.5970221161648301, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "2f3f136f2937a5b3202d8da849c2ef52e3ba1205b3daba868e9420950bc47aef", "size": 505, "artifact_path": "wandb-client-artifact://nl8mrk3hob00pmlc9wpy3nn3hz6bywndapphthnmfydu6kebgq38h8t2z6tapf0opb5vwvi6fkz45u13neef3rlgd572ubhr6za9nuwjphhotttt9xn2dwmkeqfhp2w5/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://nylkag4dcfj2agi5j8zjosvghs3bjore1u0ra1113lp8exlowxg7s1bsks6csxvdqvfj9vg3b8xp0mr6prls57raxj09a0y6mykz56ntkad51ukskxzhu2hp262c8yyg:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 350}, "eval/total_success_rate": 0.5, "eval/total_success_rate_lb_0.01": 0.33669590950012207, "eval/total_success_rate_ub_0.01": 0.6620035767555237, "eval/total_success_rate_lb_0.025": 0.3672531843185425, "eval/total_success_rate_ub_0.025": 0.6438583135604858, "eval/total_success_rate_lb_0.05": 0.3879786729812622, "eval/total_success_rate_ub_0.05": 0.6119417548179626, "eval/total_success_rate_lb_0.1": 0.4043694734573364, "eval/total_success_rate_ub_0.1": 0.5970221161842346} \ No newline at end of file diff --git a/8qih81be/wandb/run-20250101_112214-8qih81be/logs/debug-internal.log b/8qih81be/wandb/run-20250101_112214-8qih81be/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..17a32ab10d229fc155f655fef91c9e1c0ebf07b4 --- /dev/null +++ b/8qih81be/wandb/run-20250101_112214-8qih81be/logs/debug-internal.log @@ -0,0 +1,906 @@ +2025-01-01 11:22:14,855 INFO StreamThr :1793014 [internal.py:wandb_internal():85] W&B internal server running at pid: 1793014, started at: 2025-01-01 11:22:14.854579 +2025-01-01 11:22:14,857 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status +2025-01-01 11:22:14,868 INFO WriterThread:1793014 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/run-8qih81be.wandb +2025-01-01 11:22:14,868 DEBUG SenderThread:1793014 [sender.py:send():379] send: header +2025-01-01 11:22:14,876 DEBUG SenderThread:1793014 [sender.py:send():379] send: run +2025-01-01 11:22:15,125 INFO SenderThread:1793014 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files +2025-01-01 11:22:15,125 INFO SenderThread:1793014 [sender.py:_start_run_threads():1188] run started: 8qih81be with start time 1735748534.855132 +2025-01-01 11:22:15,137 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 11:22:15,137 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: check_version +2025-01-01 11:22:15,192 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 11:22:15,197 DEBUG HandlerThread:1793014 [system_info.py:__init__():26] System info init +2025-01-01 11:22:15,197 DEBUG HandlerThread:1793014 [system_info.py:__init__():41] System info init done +2025-01-01 11:22:15,197 INFO HandlerThread:1793014 [system_monitor.py:start():194] Starting system monitor +2025-01-01 11:22:15,197 INFO SystemMonitor:1793014 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 11:22:15,197 INFO HandlerThread:1793014 [system_monitor.py:probe():214] Collecting system info +2025-01-01 11:22:15,198 INFO SystemMonitor:1793014 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 11:22:15,198 INFO SystemMonitor:1793014 [interfaces.py:start():188] Started disk monitoring +2025-01-01 11:22:15,199 INFO SystemMonitor:1793014 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 11:22:15,199 INFO SystemMonitor:1793014 [interfaces.py:start():188] Started memory monitoring +2025-01-01 11:22:15,199 INFO SystemMonitor:1793014 [interfaces.py:start():188] Started network monitoring +2025-01-01 11:22:15,209 DEBUG HandlerThread:1793014 [system_info.py:probe():152] Probing system +2025-01-01 11:22:15,211 DEBUG HandlerThread:1793014 [system_info.py:_probe_git():137] Probing git +2025-01-01 11:22:15,216 DEBUG HandlerThread:1793014 [system_info.py:_probe_git():145] Probing git done +2025-01-01 11:22:15,216 DEBUG HandlerThread:1793014 [system_info.py:probe():200] Probing system done +2025-01-01 11:22:15,216 DEBUG HandlerThread:1793014 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T16:22:15.209885', 'startedAt': '2025-01-01T16:22:14.843448', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.949625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.01, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 696.8989372253418}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 11:22:15,216 INFO HandlerThread:1793014 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 11:22:15,216 INFO HandlerThread:1793014 [system_monitor.py:probe():227] Publishing system info +2025-01-01 11:22:15,216 DEBUG HandlerThread:1793014 [system_info.py:_save_code():44] Saving code +2025-01-01 11:22:15,222 DEBUG HandlerThread:1793014 [system_info.py:_save_code():67] Saving code done +2025-01-01 11:22:15,222 DEBUG HandlerThread:1793014 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 11:22:15,283 DEBUG HandlerThread:1793014 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 11:22:15,284 INFO HandlerThread:1793014 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 11:22:15,287 DEBUG SenderThread:1793014 [sender.py:send():379] send: files +2025-01-01 11:22:15,287 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 11:22:15,288 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 11:22:15,288 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 11:22:15,528 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 11:22:15,528 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: python_packages +2025-01-01 11:22:15,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:15,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:22:15,530 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:22:15,560 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/mh52qoln-diff.patch +2025-01-01 11:22:15,573 INFO HandlerThread:1793014 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/tb" + save: true +} +control { + relay_id: "f413d9117f934994a9f054bb1ef32783" +} +_info { + stream_id: "8qih81be" +} + +2025-01-01 11:22:15,573 DEBUG HandlerThread:1793014 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 11:22:15,581 INFO wandb-upload_0:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/kz92ej5k-wandb-metadata.json +2025-01-01 11:22:15,591 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/11xpddmv-code/FISH/eval_robot.py +2025-01-01 11:22:15,619 DEBUG SenderThread:1793014 [sender.py:send():379] send: telemetry +2025-01-01 11:22:15,624 DEBUG SenderThread:1793014 [sender.py:send():379] send: tbrecord +2025-01-01 11:22:15,625 DEBUG SenderThread:1793014 [sender.py:send():379] send: files +2025-01-01 11:22:15,625 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 with policy live +2025-01-01 11:22:15,810 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/svz00vyo-events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:22:16,127 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-metadata.json +2025-01-01 11:22:16,127 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/requirements.txt +2025-01-01 11:22:16,127 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/diff.patch +2025-01-01 11:22:16,128 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:22:16,128 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:16,128 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/code/FISH/eval_robot.py +2025-01-01 11:22:16,128 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/code/FISH +2025-01-01 11:22:16,128 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/code +2025-01-01 11:22:16,528 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:17,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:18,127 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:18,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:19,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:20,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:20,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:20,803 DEBUG SenderThread:1793014 [sender.py:send():379] send: config +2025-01-01 11:22:21,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:22,128 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:22,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:23,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:24,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:25,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:25,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:26,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:27,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:28,129 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:28,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:29,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:30,528 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:22:30,529 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:22:30,569 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:30,660 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:31,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:32,130 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:32,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:33,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:34,131 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:34,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:35,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:36,132 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:36,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:36,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:37,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:38,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:39,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:40,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:41,532 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:42,134 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:42,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:42,532 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:43,532 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:44,532 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:45,528 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:22:45,529 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:22:45,532 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:46,135 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:46,532 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:47,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:47,789 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:48,136 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:48,136 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/config.yaml +2025-01-01 11:22:48,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:49,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:50,137 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:50,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:51,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:52,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:53,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:53,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:54,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:55,534 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:56,138 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:22:56,534 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:57,534 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:58,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:22:58,534 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:22:59,534 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:00,139 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:00,528 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:23:00,529 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:23:00,569 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:01,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:02,534 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:03,534 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:03,619 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:04,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:05,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:06,141 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:06,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:07,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:08,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:08,843 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:09,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:10,142 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:10,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:11,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:12,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:13,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:14,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:14,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:15,200 DEBUG SystemMonitor:1793014 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 11:23:15,200 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:23:15,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:23:15,529 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:23:15,569 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:16,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:17,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:18,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:19,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:20,005 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:20,145 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:20,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:21,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:22,145 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:22,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:23,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:24,146 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:24,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:25,209 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:25,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:26,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:27,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:28,147 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:28,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:29,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:30,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:23:30,529 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:23:30,577 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:30,654 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:31,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:32,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:33,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:34,149 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:34,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:35,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:36,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:36,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:37,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:38,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:39,539 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:40,539 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:41,539 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:41,912 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:42,122 DEBUG SenderThread:1793014 [sender.py:send():379] send: files +2025-01-01 11:23:42,122 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_080cfd687bc0e6205960.mp4 with policy now +2025-01-01 11:23:42,122 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:23:42,123 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:23:42,123 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:23:42,124 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:23:42,152 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:42,152 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:23:42,152 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/0_eval_0_080cfd687bc0e6205960.mp4 +2025-01-01 11:23:42,152 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval +2025-01-01 11:23:42,152 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos +2025-01-01 11:23:42,152 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media +2025-01-01 11:23:42,350 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/608zbu6z-media/videos/eval/0_eval_0_080cfd687bc0e6205960.mp4 +2025-01-01 11:23:42,539 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:42,648 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/s5bvtm0i-media/videos/eval/0_eval_0_080cfd687bc0e6205960.mp4 +2025-01-01 11:23:43,539 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:44,152 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:44,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:45,201 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:23:45,529 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:23:45,530 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:23:45,573 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:46,540 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:47,133 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:47,541 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:48,153 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:48,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:49,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:50,541 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:51,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:52,135 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:52,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:53,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:54,155 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:54,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:55,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:56,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:57,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:57,937 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:23:58,157 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:23:58,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:23:59,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:00,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:24:00,530 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:24:00,573 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:01,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:02,543 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:03,158 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:24:03,399 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/0gi7em4o-events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:24:03,543 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:03,687 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:04,158 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:04,543 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:05,543 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:06,543 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:07,543 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:08,544 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:08,693 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:09,544 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:10,160 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:10,544 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:11,544 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:12,544 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:13,544 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:13,753 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:14,162 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:14,545 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:15,201 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:24:15,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:24:15,531 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:24:15,573 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:16,163 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:16,545 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:17,545 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:18,163 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:18,545 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:19,546 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:19,716 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:20,545 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:21,545 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:22,546 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:23,546 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:24,549 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:25,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:25,546 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:26,546 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:27,546 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:28,547 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:29,547 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:30,530 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:24:30,530 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:24:30,577 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:30,657 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:31,547 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:32,547 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:33,548 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:34,547 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:35,547 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:35,661 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:36,549 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:36,705 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:24:36,705 DEBUG SenderThread:1793014 [sender.py:send():379] send: files +2025-01-01 11:24:36,706 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_d616a9f6d337515bde74.mp4 with policy now +2025-01-01 11:24:36,706 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:24:36,706 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:24:36,706 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:24:36,915 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/6t89wdll-media/videos/eval/1_eval_1_d616a9f6d337515bde74.mp4 +2025-01-01 11:24:37,169 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:24:37,170 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/1_eval_1_d616a9f6d337515bde74.mp4 +2025-01-01 11:24:37,170 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval +2025-01-01 11:24:37,433 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/srhlcvks-media/videos/eval/1_eval_1_d616a9f6d337515bde74.mp4 +2025-01-01 11:24:37,550 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:38,170 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:38,551 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:39,556 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:40,559 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:41,539 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:41,558 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:42,554 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:43,556 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:44,172 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:44,556 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:45,202 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:24:45,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:24:45,535 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:24:45,577 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:46,558 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:46,648 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:47,557 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:48,173 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:48,568 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:49,558 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:50,174 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:50,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:51,559 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:52,174 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:24:52,559 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:52,637 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:53,559 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:54,560 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:55,559 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:56,560 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:57,560 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:58,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:24:58,560 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:24:59,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:00,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:25:00,532 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:25:00,573 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:01,560 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:02,561 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:03,561 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:03,589 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:04,561 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:05,047 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:25:05,047 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:25:05,047 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:25:05,179 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:25:05,179 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:25:05,411 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/tw9zmb98-events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:25:05,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:06,179 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:06,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:07,561 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:08,561 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:09,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:09,564 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:10,561 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:11,561 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:12,181 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:12,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:13,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:14,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:14,890 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:15,202 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:25:15,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:25:15,531 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:25:15,563 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:16,183 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:16,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:17,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:18,183 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:18,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:19,562 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:20,184 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:20,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:20,563 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:21,563 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:22,563 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:23,563 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:24,564 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:25,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:25,564 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:26,564 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:27,564 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:28,187 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:28,564 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:29,564 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:30,188 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:30,531 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:25:30,532 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:25:30,573 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:30,690 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:31,564 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:32,565 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:33,565 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:34,565 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:35,565 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:36,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:36,565 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:37,565 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:38,565 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:39,565 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:40,565 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:41,574 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:41,752 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:41,894 DEBUG SenderThread:1793014 [sender.py:send():379] send: files +2025-01-01 11:25:41,894 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_52a5d239f2ed3d88df0e.mp4 with policy now +2025-01-01 11:25:41,901 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:25:41,901 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:25:41,901 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:25:41,901 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:25:42,104 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/uu5x662e-media/videos/eval/2_eval_2_52a5d239f2ed3d88df0e.mp4 +2025-01-01 11:25:42,192 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:25:42,192 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:42,192 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/2_eval_2_52a5d239f2ed3d88df0e.mp4 +2025-01-01 11:25:42,192 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval +2025-01-01 11:25:42,454 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/1bz77e3s-media/videos/eval/2_eval_2_52a5d239f2ed3d88df0e.mp4 +2025-01-01 11:25:42,570 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:43,569 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:44,192 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:44,571 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:45,203 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:25:45,535 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:25:45,536 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:25:45,577 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:46,573 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:46,937 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:47,576 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:48,194 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:48,581 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:49,579 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:50,575 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:51,575 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:52,039 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:52,576 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:53,579 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:54,196 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:54,580 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:55,580 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:56,583 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:57,580 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:57,998 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:25:58,197 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:25:58,579 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:25:59,584 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:00,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:26:00,536 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:26:00,582 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:00,895 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:26:00,895 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:26:00,896 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:26:01,198 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:26:01,198 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:26:01,589 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:02,198 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:02,592 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:03,590 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:03,896 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:04,589 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:05,592 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:06,594 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:07,594 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:08,200 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:08,593 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:09,595 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:09,853 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:10,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:11,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:12,202 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:12,598 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:13,598 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:14,202 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:14,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:14,923 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:15,203 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:26:15,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:26:15,534 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:26:15,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:16,204 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:16,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:17,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:18,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:19,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:20,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:20,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:21,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:22,596 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:23,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:24,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:25,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:26,541 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:26,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:27,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:28,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:29,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:30,533 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:26:30,534 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:26:30,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:31,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:31,961 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:32,209 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:32,597 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:33,604 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:34,210 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:34,305 DEBUG SenderThread:1793014 [sender.py:send():379] send: files +2025-01-01 11:26:34,305 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c6b478b7570f0157dada.mp4 with policy now +2025-01-01 11:26:34,307 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:26:34,308 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:26:34,308 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:26:34,308 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:26:34,508 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/eeubojar-media/videos/eval/3_eval_3_c6b478b7570f0157dada.mp4 +2025-01-01 11:26:34,599 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:35,210 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:26:35,210 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/3_eval_3_c6b478b7570f0157dada.mp4 +2025-01-01 11:26:35,211 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval +2025-01-01 11:26:35,472 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/dmli1tfo-media/videos/eval/3_eval_3_c6b478b7570f0157dada.mp4 +2025-01-01 11:26:35,604 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:36,211 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:36,600 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:37,550 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:37,603 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:38,606 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:39,609 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:40,212 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:40,606 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:41,608 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:42,607 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:43,548 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:43,624 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:44,635 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:45,204 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:26:45,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:26:45,536 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:26:45,630 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:46,214 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:46,622 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:47,622 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:48,215 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:48,617 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:49,245 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:49,622 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:50,215 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:26:50,622 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:51,626 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:52,627 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:53,628 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:54,549 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:26:54,629 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:55,631 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:56,631 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:57,634 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:57,960 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:26:57,961 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:26:57,962 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:26:58,218 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:26:58,218 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:26:58,503 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/jv7vsvf1-events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:26:58,637 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:59,632 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:26:59,962 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:00,219 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:00,547 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:27:00,547 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:27:00,643 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:01,637 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:02,645 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:03,638 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:04,220 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:04,636 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:05,549 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:05,644 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:06,642 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:07,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:08,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:09,642 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:10,222 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:10,552 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:10,649 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:11,647 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:12,645 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:13,645 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:14,224 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:14,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:15,204 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:27:15,536 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:27:15,537 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:27:15,634 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:15,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:16,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:17,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:18,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:19,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:20,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:21,551 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:21,646 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:22,647 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:23,647 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:24,647 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:25,647 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:26,551 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:26,647 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:27,647 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:28,647 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:29,648 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:30,229 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:30,537 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:27:30,537 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:27:30,648 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:31,625 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:31,648 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:32,648 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:33,648 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:34,648 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:35,554 DEBUG SenderThread:1793014 [sender.py:send():379] send: files +2025-01-01 11:27:35,555 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_b9e904a3822ce75d6da0.mp4 with policy now +2025-01-01 11:27:35,561 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:27:35,562 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:27:35,562 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:35,562 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:35,651 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:35,769 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/9c5dleaq-media/videos/eval/4_eval_4_b9e904a3822ce75d6da0.mp4 +2025-01-01 11:27:36,231 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:27:36,231 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:36,231 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/4_eval_4_b9e904a3822ce75d6da0.mp4 +2025-01-01 11:27:36,231 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval +2025-01-01 11:27:36,491 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/54r0jryy-media/videos/eval/4_eval_4_b9e904a3822ce75d6da0.mp4 +2025-01-01 11:27:36,656 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:37,610 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:37,664 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:38,232 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:38,658 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:39,662 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:40,659 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:41,657 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:42,233 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:42,657 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:42,658 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:43,661 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:44,667 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:45,204 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:27:45,542 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:27:45,542 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:27:45,665 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:46,666 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:47,663 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:47,765 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:48,235 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:48,664 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:49,673 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:50,674 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:51,667 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:52,236 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:52,674 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:53,561 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:53,677 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:54,667 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:55,673 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:56,675 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:57,675 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:57,762 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:27:57,762 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:27:57,959 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:27:58,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:27:58,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:27:58,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:27:58,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media +2025-01-01 11:27:58,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table/eval +2025-01-01 11:27:58,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table +2025-01-01 11:27:58,243 INFO wandb-upload_2:1793014 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp3itgor1_ +2025-01-01 11:27:58,394 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:27:58,402 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:27:58,453 INFO wandb-upload_0:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/al84qfgm-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:27:58,479 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/ta6u3jjt-events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:27:58,641 INFO SenderThread:1793014 [sender.py:send_request_log_artifact():1525] logged artifact run-8qih81be-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY4Nzg2NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MjQ4NQ==', 'latestArtifact': None}} +2025-01-01 11:27:58,642 DEBUG SenderThread:1793014 [sender.py:send():379] send: files +2025-01-01 11:27:58,642 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-01 11:27:58,642 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:27:58,643 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,643 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:27:58,644 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,644 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:27:58,644 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,648 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,648 DEBUG SenderThread:1793014 [sender.py:send():379] send: telemetry +2025-01-01 11:27:58,648 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,648 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,648 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,649 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,649 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,649 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,649 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,650 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,650 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,650 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,650 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,651 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,651 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,651 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,651 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,652 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,652 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,652 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,652 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,653 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,653 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:58,653 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:58,653 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:27:58,674 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:58,873 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/j0y7gcuh-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:27:59,000 INFO wandb-upload_1:1793014 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwi4_w24g +2025-01-01 11:27:59,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:27:59,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:27:59,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-01 11:27:59,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table/eval +2025-01-01 11:27:59,402 INFO SenderThread:1793014 [sender.py:send_request_log_artifact():1525] logged artifact run-8qih81be-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY4Nzg3Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MjQ4Nw==', 'latestArtifact': None}} +2025-01-01 11:27:59,402 DEBUG SenderThread:1793014 [sender.py:send():379] send: files +2025-01-01 11:27:59,402 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json with policy now +2025-01-01 11:27:59,402 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:27:59,403 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:27:59,403 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:27:59,574 INFO wandb-upload_0:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/tlk04g5i-media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-01 11:27:59,674 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:27:59,825 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/whp3g0j5-media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-01 11:28:00,239 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:28:00,240 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:28:00,538 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:28:00,538 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:28:00,674 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:01,674 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:02,675 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:03,675 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:04,638 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:04,675 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:05,689 DEBUG SenderThread:1793014 [sender.py:send():379] send: exit +2025-01-01 11:28:05,689 INFO SenderThread:1793014 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 11:28:05,690 INFO SenderThread:1793014 [sender.py:send_exit():588] handling runtime: 350 +2025-01-01 11:28:05,690 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:28:05,690 INFO SenderThread:1793014 [sender.py:send_exit():594] send defer +2025-01-01 11:28:05,690 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:05,690 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 11:28:05,690 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:05,690 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 11:28:05,690 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 1 +2025-01-01 11:28:05,690 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:05,691 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 11:28:05,691 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:05,691 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 11:28:05,691 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 2 +2025-01-01 11:28:05,691 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:05,691 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 11:28:05,691 INFO HandlerThread:1793014 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 11:28:05,691 DEBUG SystemMonitor:1793014 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 11:28:05,691 DEBUG SystemMonitor:1793014 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 11:28:05,692 INFO HandlerThread:1793014 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 11:28:05,692 INFO HandlerThread:1793014 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 11:28:05,743 INFO HandlerThread:1793014 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 11:28:05,743 INFO HandlerThread:1793014 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 11:28:05,744 INFO HandlerThread:1793014 [interfaces.py:finish():200] Joined network monitor +2025-01-01 11:28:05,744 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:05,744 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 11:28:05,744 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 3 +2025-01-01 11:28:05,744 DEBUG SenderThread:1793014 [sender.py:send():379] send: stats +2025-01-01 11:28:05,744 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:05,744 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 11:28:05,745 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:05,745 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 11:28:05,745 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 4 +2025-01-01 11:28:05,745 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:05,745 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 11:28:06,242 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:28:08,243 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:28:12,037 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:28:12,037 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:12,037 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 11:28:12,038 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:12,038 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 11:28:12,039 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 5 +2025-01-01 11:28:12,039 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:28:12,039 DEBUG SenderThread:1793014 [sender.py:send():379] send: history +2025-01-01 11:28:12,039 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:28:12,040 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:12,040 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 11:28:12,040 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:28:12,041 DEBUG SenderThread:1793014 [sender.py:send():379] send: summary +2025-01-01 11:28:12,044 INFO SenderThread:1793014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:28:12,045 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:12,045 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 11:28:12,045 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 6 +2025-01-01 11:28:12,045 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:12,045 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 11:28:12,045 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:12,045 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 11:28:12,049 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:12,171 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 7 +2025-01-01 11:28:12,171 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:12,171 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 11:28:12,171 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:12,171 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 11:28:12,244 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:28:12,245 INFO Thread-12 :1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/config.yaml +2025-01-01 11:28:12,690 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:28:15,879 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 8 +2025-01-01 11:28:15,880 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:28:15,880 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:15,880 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 11:28:15,880 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:15,880 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 11:28:15,880 INFO SenderThread:1793014 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 11:28:15,883 INFO SenderThread:1793014 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 11:28:15,884 INFO SenderThread:1793014 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 11:28:15,898 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 9 +2025-01-01 11:28:15,898 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:15,898 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 11:28:15,898 DEBUG SenderThread:1793014 [sender.py:send():379] send: artifact +2025-01-01 11:28:16,151 INFO SenderThread:1793014 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjk2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 11:28:16,151 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:16,151 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 11:28:16,151 INFO SenderThread:1793014 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 11:28:16,245 INFO SenderThread:1793014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:28:16,246 INFO SenderThread:1793014 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files +2025-01-01 11:28:16,246 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/config.yaml config.yaml +2025-01-01 11:28:16,246 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json wandb-summary.json +2025-01-01 11:28:16,246 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/requirements.txt requirements.txt +2025-01-01 11:28:16,246 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-metadata.json wandb-metadata.json +2025-01-01 11:28:16,246 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:28:16,249 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log output.log +2025-01-01 11:28:16,249 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/diff.patch diff.patch +2025-01-01 11:28:16,251 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 11:28:16,251 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/1_eval_1_d616a9f6d337515bde74.mp4 media/videos/eval/1_eval_1_d616a9f6d337515bde74.mp4 +2025-01-01 11:28:16,251 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/4_eval_4_b9e904a3822ce75d6da0.mp4 media/videos/eval/4_eval_4_b9e904a3822ce75d6da0.mp4 +2025-01-01 11:28:16,251 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/3_eval_3_c6b478b7570f0157dada.mp4 media/videos/eval/3_eval_3_c6b478b7570f0157dada.mp4 +2025-01-01 11:28:16,251 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/2_eval_2_52a5d239f2ed3d88df0e.mp4 media/videos/eval/2_eval_2_52a5d239f2ed3d88df0e.mp4 +2025-01-01 11:28:16,252 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/videos/eval/0_eval_0_080cfd687bc0e6205960.mp4 media/videos/eval/0_eval_0_080cfd687bc0e6205960.mp4 +2025-01-01 11:28:16,253 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-01 11:28:16,253 INFO SenderThread:1793014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:28:16,253 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 10 +2025-01-01 11:28:16,253 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:16,253 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 11:28:16,254 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:16,254 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 11:28:16,254 INFO SenderThread:1793014 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:28:16,436 INFO wandb-upload_1:1793014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/requirements.txt +2025-01-01 11:28:16,441 INFO wandb-upload_2:1793014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/wandb-summary.json +2025-01-01 11:28:16,462 INFO wandb-upload_0:1793014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/config.yaml +2025-01-01 11:28:16,491 INFO wandb-upload_4:1793014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/files/output.log +2025-01-01 11:28:16,548 INFO wandb-upload_3:1793014 [upload_job.py:push():130] Uploaded file /tmp/tmprigw3vgjwandb/nbsw5bpi-events.out.tfevents.1735748535.leonmkim-ROG-Strix-G15CS-G15CS.1792840.0 +2025-01-01 11:28:16,692 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:28:16,692 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:28:16,748 INFO Thread-11 (_thread_body):1793014 [sender.py:transition_state():614] send defer: 11 +2025-01-01 11:28:16,748 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:16,748 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 11:28:16,749 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:16,749 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 11:28:16,749 INFO SenderThread:1793014 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 11:28:16,750 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 12 +2025-01-01 11:28:16,750 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:16,750 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 11:28:16,750 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:16,750 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 11:28:16,750 INFO SenderThread:1793014 [file_stream.py:finish():601] file stream finish called +2025-01-01 11:28:16,968 INFO SenderThread:1793014 [file_stream.py:finish():605] file stream finish is done +2025-01-01 11:28:16,968 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 13 +2025-01-01 11:28:16,968 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:16,968 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 11:28:16,968 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:16,968 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 11:28:16,968 INFO SenderThread:1793014 [sender.py:transition_state():614] send defer: 14 +2025-01-01 11:28:16,968 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:28:16,968 INFO HandlerThread:1793014 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 11:28:16,969 DEBUG SenderThread:1793014 [sender.py:send():379] send: final +2025-01-01 11:28:16,969 DEBUG SenderThread:1793014 [sender.py:send():379] send: footer +2025-01-01 11:28:16,969 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: defer +2025-01-01 11:28:16,969 INFO SenderThread:1793014 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 11:28:16,969 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:28:16,969 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:28:16,970 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 11:28:16,970 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:28:16,970 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 11:28:16,970 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: server_info +2025-01-01 11:28:16,971 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 11:28:16,973 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:17,032 DEBUG SenderThread:1793014 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:28:17,033 INFO MainThread:1793014 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 11:28:17,033 INFO MainThread:1793014 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 11:28:17,033 INFO MainThread:1793014 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 11:28:17,034 DEBUG HandlerThread:1793014 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 11:28:17,034 INFO HandlerThread:1793014 [handler.py:finish():882] shutting down handler +2025-01-01 11:28:17,970 INFO WriterThread:1793014 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/run-8qih81be.wandb +2025-01-01 11:28:18,033 INFO SenderThread:1793014 [sender.py:finish():1615] shutting down sender +2025-01-01 11:28:18,033 INFO SenderThread:1793014 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:28:18,033 INFO SenderThread:1793014 [file_pusher.py:join():175] waiting for file pusher diff --git a/8qih81be/wandb/run-20250101_112214-8qih81be/logs/debug.log b/8qih81be/wandb/run-20250101_112214-8qih81be/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e0140b5afaeabfea4e7ead39a34f7c03eb2b1f7b --- /dev/null +++ b/8qih81be/wandb/run-20250101_112214-8qih81be/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_setup.py:_flush():76] Configure stats pid to 1792840 +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/settings +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/logs/debug.log +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/wandb/run-20250101_112214-8qih81be/logs/debug-internal.log +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_init.py:init():569] calling init triggers +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be'} +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_init.py:init():619] starting backend +2025-01-01 11:22:14,847 INFO MainThread:1792840 [wandb_init.py:init():623] setting up manager +2025-01-01 11:22:14,853 INFO MainThread:1792840 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 11:22:14,854 INFO MainThread:1792840 [wandb_init.py:init():631] backend started and connected +2025-01-01 11:22:14,868 INFO MainThread:1792840 [wandb_init.py:init():720] updated telemetry +2025-01-01 11:22:14,874 INFO MainThread:1792840 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 11:22:15,136 INFO MainThread:1792840 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 11:22:15,186 INFO MainThread:1792840 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 11:22:15,186 INFO MainThread:1792840 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 11:22:15,528 INFO MainThread:1792840 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 11:22:15,529 INFO MainThread:1792840 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 11:22:15,529 INFO MainThread:1792840 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 11:22:15,529 INFO MainThread:1792840 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 11:22:15,531 INFO MainThread:1792840 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 11:22:15,531 INFO MainThread:1792840 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/8qih81be/tb, True +2025-01-01 11:22:20,803 INFO MainThread:1792840 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-01 11:28:18,034 WARNING MsgRouterThr:1792840 [router.py:message_loop():77] message_loop has been closed diff --git a/8qih81be/wandb/run-20250101_112214-8qih81be/run-8qih81be.wandb b/8qih81be/wandb/run-20250101_112214-8qih81be/run-8qih81be.wandb new file mode 100644 index 0000000000000000000000000000000000000000..6aef246f14d9227064a4fdf2af769e26a1f27a51 Binary files /dev/null and b/8qih81be/wandb/run-20250101_112214-8qih81be/run-8qih81be.wandb differ diff --git a/c7zbixne/episode_rosbags/episode_0_2024-12-31-21-35-34.bag b/c7zbixne/episode_rosbags/episode_0_2024-12-31-21-35-34.bag new file mode 100644 index 0000000000000000000000000000000000000000..b6febb973609319870298822c420750af43a8856 --- /dev/null +++ b/c7zbixne/episode_rosbags/episode_0_2024-12-31-21-35-34.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ede20c9a6f5768176ae9f9d2bee43865c49b8eaf199db6a0b57be35ba5774f92 +size 2617262452 diff --git a/c7zbixne/episode_rosbags/episode_3_2024-12-31-21-38-07.bag b/c7zbixne/episode_rosbags/episode_3_2024-12-31-21-38-07.bag new file mode 100644 index 0000000000000000000000000000000000000000..cc29ca0f7a3b077fa1309777c1fca82a88ef506b --- /dev/null +++ b/c7zbixne/episode_rosbags/episode_3_2024-12-31-21-38-07.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:751f8ee8276882ef9c96cf29cb3fe2eb818d7764b9a45a75492481a21d0a558a +size 2616227718 diff --git a/c7zbixne/episode_rosbags/episode_4_2024-12-31-21-39-09.bag b/c7zbixne/episode_rosbags/episode_4_2024-12-31-21-39-09.bag new file mode 100644 index 0000000000000000000000000000000000000000..66479db4537e9e1c89974b04fa56902eadd31768 --- /dev/null +++ b/c7zbixne/episode_rosbags/episode_4_2024-12-31-21-39-09.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cfca754629b71c6b5e29c3af8072b901dddc0e7f807985b224ae83699a5c2f7 +size 2609871725 diff --git a/hex7gk4r/.hydra/config.yaml b/hex7gk4r/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05485f98b3c08bece875a460fe25a0cce7504f90 --- /dev/null +++ b/hex7gk4r/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1006_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/hex7gk4r/.hydra/hydra.yaml b/hex7gk4r/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ec5051c9ffc3eee80723c38f19a93eb6cdc3374 --- /dev/null +++ b/hex7gk4r/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/hex7gk4r/.hydra/overrides.yaml b/hex7gk4r/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/hex7gk4r/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/hex7gk4r/episode_rosbags/aligned_depth_to_color_K.npy b/hex7gk4r/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/hex7gk4r/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/hex7gk4r/episode_rosbags/cam_tf_world.npy b/hex7gk4r/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/hex7gk4r/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/hex7gk4r/episode_rosbags/color_K.npy b/hex7gk4r/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/hex7gk4r/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/hex7gk4r/episode_rosbags/depth_K.npy b/hex7gk4r/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/hex7gk4r/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/hex7gk4r/eval_robot.log b/hex7gk4r/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..bf8efdfaa31201f2188f00fb899b8fc364a7c14f --- /dev/null +++ b/hex7gk4r/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-01 11:02:57,296][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 11:02:57,299][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/hex7gk4r/eval_video/0_eval.mp4 b/hex7gk4r/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..434cba8df6afaa36901fc899837c2bec9c02756a --- /dev/null +++ b/hex7gk4r/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a25a53b5c8820b669aa14051380155d22a72decf1d1e0caeb8374cbec73635b +size 1476161 diff --git a/hex7gk4r/eval_video/1_eval.mp4 b/hex7gk4r/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a701bae63751fe7d2a01a649654639170a0b422f --- /dev/null +++ b/hex7gk4r/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fae1d5a4084c3fa7e4ce9bcd488a835aef4c52ce2bc8f173c5556165eb635f28 +size 1463141 diff --git a/hex7gk4r/eval_video/2_eval.mp4 b/hex7gk4r/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ddfd806dd7114c9d6c34daf99d5dddea5f1e02b8 --- /dev/null +++ b/hex7gk4r/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07f9274f756bd81cbaeea939af0ed857deb6ec61b615f296fd45aa9ca85a9ed0 +size 1471142 diff --git a/hex7gk4r/eval_video/3_eval.mp4 b/hex7gk4r/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..58d7a1a8c554017dbdca0d81a0271cfbb0ce3403 --- /dev/null +++ b/hex7gk4r/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26ea14ca5e0897c83196858a6de03ff39a6472b34e7de7ed4e68df94cb194a95 +size 1458359 diff --git a/hex7gk4r/eval_video/4_eval.mp4 b/hex7gk4r/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ef64b79cd119d10ded6ea4a77d94991ceb4ff686 --- /dev/null +++ b/hex7gk4r/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20669f7dea9bec0e6b764682fffadaf458c7316efe766c00e0e5648e29d8046c +size 1487542 diff --git a/hex7gk4r/tb/events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 b/hex7gk4r/tb/events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 new file mode 100644 index 0000000000000000000000000000000000000000..cf3ba126f5036494032b4ad71246b6bf56b5834a --- /dev/null +++ b/hex7gk4r/tb/events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c54139dba1f8da8476c01950491271ac295ecaa18c595481fd67ddbfff33ea53 +size 6539 diff --git a/hex7gk4r/wandb/debug-internal.log b/hex7gk4r/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7ef422b0ddd046526174c27432863f831939048c --- /dev/null +++ b/hex7gk4r/wandb/debug-internal.log @@ -0,0 +1,845 @@ +2025-01-01 11:03:03,231 INFO StreamThr :1789646 [internal.py:wandb_internal():85] W&B internal server running at pid: 1789646, started at: 2025-01-01 11:03:03.230653 +2025-01-01 11:03:03,232 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status +2025-01-01 11:03:03,245 INFO WriterThread:1789646 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/run-hex7gk4r.wandb +2025-01-01 11:03:03,246 DEBUG SenderThread:1789646 [sender.py:send():379] send: header +2025-01-01 11:03:03,256 DEBUG SenderThread:1789646 [sender.py:send():379] send: run +2025-01-01 11:03:03,488 INFO SenderThread:1789646 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files +2025-01-01 11:03:03,488 INFO SenderThread:1789646 [sender.py:_start_run_threads():1188] run started: hex7gk4r with start time 1735747383.230972 +2025-01-01 11:03:03,501 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 11:03:03,501 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: check_version +2025-01-01 11:03:03,570 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 11:03:03,575 DEBUG HandlerThread:1789646 [system_info.py:__init__():26] System info init +2025-01-01 11:03:03,575 DEBUG HandlerThread:1789646 [system_info.py:__init__():41] System info init done +2025-01-01 11:03:03,575 INFO HandlerThread:1789646 [system_monitor.py:start():194] Starting system monitor +2025-01-01 11:03:03,576 INFO SystemMonitor:1789646 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 11:03:03,576 INFO HandlerThread:1789646 [system_monitor.py:probe():214] Collecting system info +2025-01-01 11:03:03,577 INFO SystemMonitor:1789646 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 11:03:03,577 INFO SystemMonitor:1789646 [interfaces.py:start():188] Started disk monitoring +2025-01-01 11:03:03,578 INFO SystemMonitor:1789646 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 11:03:03,578 INFO SystemMonitor:1789646 [interfaces.py:start():188] Started memory monitoring +2025-01-01 11:03:03,579 INFO SystemMonitor:1789646 [interfaces.py:start():188] Started network monitoring +2025-01-01 11:03:03,590 DEBUG HandlerThread:1789646 [system_info.py:probe():152] Probing system +2025-01-01 11:03:03,591 DEBUG HandlerThread:1789646 [system_info.py:_probe_git():137] Probing git +2025-01-01 11:03:03,596 DEBUG HandlerThread:1789646 [system_info.py:_probe_git():145] Probing git done +2025-01-01 11:03:03,596 DEBUG HandlerThread:1789646 [system_info.py:probe():200] Probing system done +2025-01-01 11:03:03,596 DEBUG HandlerThread:1789646 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T16:03:03.590194', 'startedAt': '2025-01-01T16:03:03.217444', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.171875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.942, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 664.0837097167969}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 11:03:03,596 INFO HandlerThread:1789646 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 11:03:03,596 INFO HandlerThread:1789646 [system_monitor.py:probe():227] Publishing system info +2025-01-01 11:03:03,596 DEBUG HandlerThread:1789646 [system_info.py:_save_code():44] Saving code +2025-01-01 11:03:03,598 DEBUG HandlerThread:1789646 [system_info.py:_save_code():67] Saving code done +2025-01-01 11:03:03,598 DEBUG HandlerThread:1789646 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 11:03:03,659 DEBUG HandlerThread:1789646 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 11:03:03,659 INFO HandlerThread:1789646 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 11:03:03,662 DEBUG SenderThread:1789646 [sender.py:send():379] send: files +2025-01-01 11:03:03,662 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 11:03:03,663 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 11:03:03,663 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 11:03:03,901 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 11:03:03,901 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: python_packages +2025-01-01 11:03:03,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:03:03,902 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:03:03,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:03,903 INFO HandlerThread:1789646 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/tb" + save: true +} +control { + relay_id: "9383c060aecf4fb1bae46e64e533ff79" +} +_info { + stream_id: "hex7gk4r" +} + +2025-01-01 11:03:03,904 DEBUG HandlerThread:1789646 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 11:03:03,911 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/m540yw5f-wandb-metadata.json +2025-01-01 11:03:03,917 INFO wandb-upload_1:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/5hwe7lky-code/FISH/eval_robot.py +2025-01-01 11:03:03,917 INFO wandb-upload_2:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/r9xivxet-diff.patch +2025-01-01 11:03:04,005 DEBUG SenderThread:1789646 [sender.py:send():379] send: telemetry +2025-01-01 11:03:04,005 DEBUG SenderThread:1789646 [sender.py:send():379] send: tbrecord +2025-01-01 11:03:04,005 DEBUG SenderThread:1789646 [sender.py:send():379] send: files +2025-01-01 11:03:04,005 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 with policy live +2025-01-01 11:03:04,206 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/el9lws25-events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:03:04,489 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/requirements.txt +2025-01-01 11:03:04,490 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:03:04,490 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:03:04,490 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/diff.patch +2025-01-01 11:03:04,490 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/code/FISH/eval_robot.py +2025-01-01 11:03:04,490 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-metadata.json +2025-01-01 11:03:04,490 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/code/FISH +2025-01-01 11:03:04,490 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/code +2025-01-01 11:03:04,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:05,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:06,489 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:03:06,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:07,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:08,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:08,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:09,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:10,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:11,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:12,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:13,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:13,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:14,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:15,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:16,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:17,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:18,901 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:03:18,901 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:03:18,945 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:18,960 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:19,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:20,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:21,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:22,904 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:23,904 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:24,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:24,904 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:25,904 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:26,904 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:27,698 DEBUG SenderThread:1789646 [sender.py:send():379] send: config +2025-01-01 11:03:27,904 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:28,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:29,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:29,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:30,495 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:03:30,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:31,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:32,496 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:03:32,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:33,901 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:03:33,902 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:03:33,945 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:34,496 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:03:34,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:34,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:35,496 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/config.yaml +2025-01-01 11:03:35,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:36,497 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:03:36,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:37,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:38,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:39,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:40,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:40,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:41,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:42,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:43,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:44,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:45,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:45,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:46,908 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:47,907 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:48,901 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:03:48,902 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:03:48,945 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:49,907 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:50,908 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:51,653 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:51,908 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:52,501 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:03:52,908 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:53,908 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:54,502 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:03:54,908 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:55,909 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:56,502 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:03:56,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:56,911 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:57,910 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:58,910 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:59,910 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:00,910 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:01,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:01,910 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:02,911 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:03,579 DEBUG SystemMonitor:1789646 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 11:04:03,579 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:04:03,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:04:03,902 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:04:03,945 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:04,911 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:05,911 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:06,911 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:07,003 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:07,911 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:08,911 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:09,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:10,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:11,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:12,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:12,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:13,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:14,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:15,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:16,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:17,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:18,790 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:18,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:04:18,902 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:04:18,945 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:19,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:20,508 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:04:20,913 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:21,913 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:22,913 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:23,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:23,913 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:24,913 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:25,913 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:26,913 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:27,913 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:28,910 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:28,914 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:29,382 DEBUG SenderThread:1789646 [sender.py:send():379] send: files +2025-01-01 11:04:29,382 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:04:29,382 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_0a25a53b5c8820b669aa.mp4 with policy now +2025-01-01 11:04:29,383 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:04:29,383 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:04:29,384 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:04:29,511 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/0_eval_0_0a25a53b5c8820b669aa.mp4 +2025-01-01 11:04:29,511 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:04:29,511 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media +2025-01-01 11:04:29,511 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval +2025-01-01 11:04:29,511 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos +2025-01-01 11:04:29,653 INFO wandb-upload_1:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/ob5kp906-media/videos/eval/0_eval_0_0a25a53b5c8820b669aa.mp4 +2025-01-01 11:04:29,914 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:29,951 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/72hg7fzg-media/videos/eval/0_eval_0_0a25a53b5c8820b669aa.mp4 +2025-01-01 11:04:30,511 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:04:30,914 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:31,915 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:32,915 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:33,580 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:04:33,904 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:04:33,904 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:04:33,915 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:34,005 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:34,915 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:35,919 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:36,513 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:04:36,915 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:37,915 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:38,514 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:04:38,915 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:39,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:39,916 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:40,916 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:41,916 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:42,916 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:43,916 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:44,916 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:45,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:45,916 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:46,916 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:47,917 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:48,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:04:48,903 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:04:48,949 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:49,917 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:50,913 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:50,961 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:51,518 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:04:51,759 INFO wandb-upload_1:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/l69935fl-events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:04:51,917 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:52,518 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:04:52,917 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:53,917 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:54,519 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:04:54,918 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:55,918 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:56,520 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:04:56,886 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:56,918 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:57,919 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:58,919 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:59,919 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:00,919 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:01,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:01,919 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:02,919 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:03,580 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:05:03,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:05:03,903 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:05:03,949 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:04,919 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:05,919 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:06,920 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:06,987 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:07,920 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:08,920 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:09,920 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:10,920 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:11,920 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:12,907 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:12,920 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:13,920 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:14,920 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:15,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:16,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:17,907 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:17,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:18,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:05:18,903 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:05:18,945 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:19,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:20,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:21,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:22,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:23,030 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:23,922 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:24,922 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:25,922 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:26,922 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:27,922 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:28,907 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:28,922 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:29,923 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:30,531 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:05:30,819 DEBUG SenderThread:1789646 [sender.py:send():379] send: files +2025-01-01 11:05:30,820 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_fae1d5a4084c3fa7e4ce.mp4 with policy now +2025-01-01 11:05:30,820 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:05:30,821 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:05:30,821 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:05:30,821 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:05:30,923 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:31,081 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/aq2sgdsa-media/videos/eval/1_eval_1_fae1d5a4084c3fa7e4ce.mp4 +2025-01-01 11:05:31,531 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:05:31,532 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/1_eval_1_fae1d5a4084c3fa7e4ce.mp4 +2025-01-01 11:05:31,532 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval +2025-01-01 11:05:31,807 INFO wandb-upload_1:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/3wtb49tu-media/videos/eval/1_eval_1_fae1d5a4084c3fa7e4ce.mp4 +2025-01-01 11:05:31,923 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:32,532 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:05:32,925 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:33,532 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:05:33,581 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:05:33,909 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:05:33,910 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:05:33,953 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:34,067 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:34,927 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:35,925 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:36,533 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:05:36,930 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:37,925 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:38,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:39,908 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:39,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:40,534 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:05:40,535 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:05:40,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:41,474 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:05:41,474 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:05:41,474 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:05:41,535 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:05:41,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:42,535 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:05:42,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:43,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:44,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:45,395 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:45,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:46,536 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:05:46,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:47,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:48,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:05:48,905 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:05:48,949 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:49,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:50,927 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:50,965 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:51,927 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:52,927 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:53,927 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:54,927 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:55,927 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:56,578 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:56,927 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:57,927 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:58,540 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:05:58,931 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:59,928 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:00,541 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:06:00,928 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:01,910 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:01,928 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:02,928 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:03,581 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:06:03,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:06:03,905 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:06:03,949 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:04,929 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:05,929 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:06,929 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:07,022 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:07,929 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:08,929 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:09,929 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:10,930 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:11,930 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:12,910 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:12,930 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:13,930 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:14,930 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:15,930 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:16,930 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:17,911 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:17,930 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:18,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:06:18,905 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:06:18,953 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:19,933 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:20,454 DEBUG SenderThread:1789646 [sender.py:send():379] send: files +2025-01-01 11:06:20,454 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_07f9274f756bd81cbaee.mp4 with policy now +2025-01-01 11:06:20,455 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:06:20,455 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:06:20,456 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:06:20,471 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:06:20,548 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:06:20,548 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:06:20,548 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/2_eval_2_07f9274f756bd81cbaee.mp4 +2025-01-01 11:06:20,548 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval +2025-01-01 11:06:20,708 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/okcuosdj-media/videos/eval/2_eval_2_07f9274f756bd81cbaee.mp4 +2025-01-01 11:06:20,931 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:20,965 INFO wandb-upload_1:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/lfz8ukdf-media/videos/eval/2_eval_2_07f9274f756bd81cbaee.mp4 +2025-01-01 11:06:21,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:22,548 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:06:22,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:23,476 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:23,549 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:06:23,936 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:24,942 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:25,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:26,550 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:06:26,940 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:27,937 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:28,918 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:28,938 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:29,942 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:30,235 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:06:30,235 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:06:30,235 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:06:30,551 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:06:30,551 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:06:30,551 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:06:30,785 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/0kxa3lse-events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:06:30,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:31,940 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:32,552 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:06:32,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:33,582 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:06:33,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:06:33,905 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:06:33,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:33,995 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:34,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:35,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:36,553 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:06:36,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:37,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:38,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:39,915 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:39,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:40,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:41,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:42,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:43,942 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:44,915 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:44,942 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:45,942 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:46,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:47,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:48,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:06:48,905 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:06:48,949 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:49,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:50,013 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:50,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:51,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:52,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:53,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:54,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:55,550 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:55,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:56,560 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:06:56,944 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:57,944 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:58,944 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:59,944 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:00,944 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:01,093 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:01,944 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:02,562 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:02,944 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:03,582 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:07:03,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:07:03,906 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:07:03,949 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:04,944 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:05,944 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:06,946 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:07,052 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:07,946 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:08,949 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:09,684 DEBUG SenderThread:1789646 [sender.py:send():379] send: files +2025-01-01 11:07:09,684 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_26ea14ca5e0897c83196.mp4 with policy now +2025-01-01 11:07:09,687 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:07:09,688 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:07:09,688 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:07:09,734 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:07:09,933 INFO wandb-upload_1:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/m2mc05m1-media/videos/eval/3_eval_3_26ea14ca5e0897c83196.mp4 +2025-01-01 11:07:09,949 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:10,565 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:10,565 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:07:10,565 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/3_eval_3_26ea14ca5e0897c83196.mp4 +2025-01-01 11:07:10,565 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval +2025-01-01 11:07:10,876 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/uxyl7jzq-media/videos/eval/3_eval_3_26ea14ca5e0897c83196.mp4 +2025-01-01 11:07:10,950 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:11,953 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:12,735 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:12,953 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:13,960 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:14,962 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:15,957 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:16,567 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:16,960 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:17,964 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:18,009 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:18,567 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:18,917 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:07:18,917 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:07:18,962 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:19,966 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:20,960 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:21,960 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:22,961 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:23,841 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:23,956 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:24,302 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:07:24,302 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:07:24,302 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:07:24,569 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:07:24,569 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:07:24,964 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:25,969 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:26,570 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:26,971 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:27,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:28,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:28,963 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:29,975 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:30,572 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:30,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:31,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:32,572 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:32,966 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:33,583 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:07:33,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:07:33,906 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:07:33,966 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:34,015 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:34,574 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:34,966 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:35,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:36,574 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:36,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:37,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:38,575 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:38,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:39,922 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:39,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:40,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:41,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:42,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:43,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:44,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:45,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:45,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:46,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:47,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:48,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:07:48,906 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:07:48,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:49,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:50,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:50,969 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:51,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:52,969 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:53,971 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:54,971 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:55,971 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:56,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:56,972 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:57,971 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:58,972 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:59,972 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:00,972 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:01,972 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:02,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:08:02,972 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:03,583 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:08:03,907 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:08:03,907 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:08:03,972 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:04,973 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:05,973 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:06,973 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:07,977 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:08,407 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:08:08,587 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:08:08,839 DEBUG SenderThread:1789646 [sender.py:send():379] send: files +2025-01-01 11:08:08,839 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_20669f7dea9bec0e6b76.mp4 with policy now +2025-01-01 11:08:08,841 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:08:08,842 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:08:08,842 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:08,842 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:08,978 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:09,094 INFO wandb-upload_1:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/5gyuhmzl-media/videos/eval/4_eval_4_20669f7dea9bec0e6b76.mp4 +2025-01-01 11:08:09,587 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:08:09,587 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/4_eval_4_20669f7dea9bec0e6b76.mp4 +2025-01-01 11:08:09,588 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval +2025-01-01 11:08:09,869 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/2fj0xs0t-media/videos/eval/4_eval_4_20669f7dea9bec0e6b76.mp4 +2025-01-01 11:08:09,981 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:10,587 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:08:10,988 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:11,995 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:12,984 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:13,514 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:08:13,996 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:14,589 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:08:14,988 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:15,994 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:16,993 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:17,993 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:18,590 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:08:18,922 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:08:18,923 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:08:18,992 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:08:18,992 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:19,996 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:20,995 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:21,995 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:23,014 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:24,005 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:24,159 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:08:24,159 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:08:24,298 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:08:24,592 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:08:24,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:08:24,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:08:24,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media +2025-01-01 11:08:24,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table/eval +2025-01-01 11:08:24,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table +2025-01-01 11:08:24,650 INFO wandb-upload_1:1789646 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpnor_r9wg +2025-01-01 11:08:24,693 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:08:24,701 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:08:24,769 INFO wandb-upload_2:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/8rhbygf3-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:08:24,841 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/izvtcfdg-events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:08:24,998 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:25,052 INFO SenderThread:1789646 [sender.py:send_request_log_artifact():1525] logged artifact run-hex7gk4r-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY3NDAzMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MDIwNg==', 'latestArtifact': None}} +2025-01-01 11:08:25,052 DEBUG SenderThread:1789646 [sender.py:send():379] send: files +2025-01-01 11:08:25,052 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-01 11:08:25,053 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:08:25,053 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:08:25,053 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,054 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,055 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:08:25,055 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,058 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,059 DEBUG SenderThread:1789646 [sender.py:send():379] send: telemetry +2025-01-01 11:08:25,059 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,059 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,059 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,059 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,060 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,060 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,060 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,060 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,060 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,061 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,061 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,061 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,061 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,062 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,062 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,062 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,062 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,063 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,063 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,063 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,063 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,063 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,064 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:08:25,297 INFO wandb-upload_1:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/8eoze2vw-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:08:25,437 INFO wandb-upload_0:1789646 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpz1gbqcms +2025-01-01 11:08:25,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:08:25,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:08:25,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json +2025-01-01 11:08:25,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table/eval +2025-01-01 11:08:25,804 INFO SenderThread:1789646 [sender.py:send_request_log_artifact():1525] logged artifact run-hex7gk4r-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY3NDA0Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MDIxMQ==', 'latestArtifact': None}} +2025-01-01 11:08:25,804 DEBUG SenderThread:1789646 [sender.py:send():379] send: files +2025-01-01 11:08:25,804 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json with policy now +2025-01-01 11:08:25,804 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:08:25,804 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,805 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,805 DEBUG SenderThread:1789646 [sender.py:send():379] send: exit +2025-01-01 11:08:25,805 INFO SenderThread:1789646 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 11:08:25,805 INFO SenderThread:1789646 [sender.py:send_exit():588] handling runtime: 321 +2025-01-01 11:08:25,806 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,806 INFO SenderThread:1789646 [sender.py:send_exit():594] send defer +2025-01-01 11:08:25,806 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:25,806 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 11:08:25,806 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:25,806 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 11:08:25,806 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 1 +2025-01-01 11:08:25,806 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:25,807 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 11:08:25,807 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:25,807 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 11:08:25,807 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 2 +2025-01-01 11:08:25,807 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:25,807 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 11:08:25,807 INFO HandlerThread:1789646 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 11:08:25,807 DEBUG SystemMonitor:1789646 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 11:08:25,807 INFO HandlerThread:1789646 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 11:08:25,807 DEBUG SystemMonitor:1789646 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 11:08:25,808 INFO HandlerThread:1789646 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 11:08:25,860 INFO HandlerThread:1789646 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 11:08:25,860 INFO HandlerThread:1789646 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 11:08:25,861 INFO HandlerThread:1789646 [interfaces.py:finish():200] Joined network monitor +2025-01-01 11:08:25,861 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:25,861 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 11:08:25,861 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 3 +2025-01-01 11:08:25,861 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:08:25,861 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:25,861 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 11:08:25,862 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:25,862 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 11:08:25,862 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 4 +2025-01-01 11:08:25,862 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:25,862 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 11:08:25,976 INFO wandb-upload_2:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/fwtuz5zp-media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json +2025-01-01 11:08:26,223 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/6n7joruu-media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json +2025-01-01 11:08:26,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:08:26,594 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:08:31,595 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/config.yaml +2025-01-01 11:08:32,380 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:08:32,380 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:32,380 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 11:08:32,380 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:08:32,380 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 5 +2025-01-01 11:08:32,380 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 11:08:32,380 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:08:32,381 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:32,382 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 11:08:32,382 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:08:32,382 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:32,383 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:32,383 DEBUG SenderThread:1789646 [sender.py:send():379] send: summary +2025-01-01 11:08:32,387 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:32,387 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:32,387 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 11:08:32,387 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 6 +2025-01-01 11:08:32,387 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:32,387 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 11:08:32,387 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:32,387 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 11:08:32,388 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 7 +2025-01-01 11:08:32,388 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:08:32,388 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:32,388 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 11:08:32,388 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:32,388 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 11:08:32,596 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:08:33,227 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:08:34,935 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 8 +2025-01-01 11:08:34,935 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:08:34,936 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:34,936 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 11:08:34,936 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:34,936 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 11:08:34,936 INFO SenderThread:1789646 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 11:08:34,938 INFO SenderThread:1789646 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 11:08:34,940 INFO SenderThread:1789646 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 11:08:34,949 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 9 +2025-01-01 11:08:34,950 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:34,950 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 11:08:34,950 DEBUG SenderThread:1789646 [sender.py:send():379] send: artifact +2025-01-01 11:08:35,133 INFO SenderThread:1789646 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjk2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 11:08:35,133 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:35,133 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 11:08:35,133 INFO SenderThread:1789646 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 11:08:35,227 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:08:35,597 INFO SenderThread:1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:08:35,597 INFO SenderThread:1789646 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files +2025-01-01 11:08:35,597 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/config.yaml config.yaml +2025-01-01 11:08:35,597 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json wandb-summary.json +2025-01-01 11:08:35,597 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:08:35,598 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/requirements.txt requirements.txt +2025-01-01 11:08:35,598 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-metadata.json wandb-metadata.json +2025-01-01 11:08:35,598 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log output.log +2025-01-01 11:08:35,600 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/diff.patch diff.patch +2025-01-01 11:08:35,602 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 11:08:35,606 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/3_eval_3_26ea14ca5e0897c83196.mp4 media/videos/eval/3_eval_3_26ea14ca5e0897c83196.mp4 +2025-01-01 11:08:35,606 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/4_eval_4_20669f7dea9bec0e6b76.mp4 media/videos/eval/4_eval_4_20669f7dea9bec0e6b76.mp4 +2025-01-01 11:08:35,606 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/1_eval_1_fae1d5a4084c3fa7e4ce.mp4 media/videos/eval/1_eval_1_fae1d5a4084c3fa7e4ce.mp4 +2025-01-01 11:08:35,606 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/0_eval_0_0a25a53b5c8820b669aa.mp4 media/videos/eval/0_eval_0_0a25a53b5c8820b669aa.mp4 +2025-01-01 11:08:35,606 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/2_eval_2_07f9274f756bd81cbaee.mp4 media/videos/eval/2_eval_2_07f9274f756bd81cbaee.mp4 +2025-01-01 11:08:35,606 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json +2025-01-01 11:08:35,607 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:08:35,607 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 10 +2025-01-01 11:08:35,607 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:08:35,607 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:35,607 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 11:08:35,607 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:35,607 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 11:08:35,607 INFO SenderThread:1789646 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:08:35,776 INFO wandb-upload_1:1789646 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:08:35,781 INFO wandb-upload_2:1789646 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/config.yaml +2025-01-01 11:08:35,828 INFO wandb-upload_3:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/lj4wjqap-events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:08:35,833 INFO wandb-upload_4:1789646 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:08:35,834 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/requirements.txt +2025-01-01 11:08:36,034 INFO Thread-11 (_thread_body):1789646 [sender.py:transition_state():614] send defer: 11 +2025-01-01 11:08:36,034 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:36,035 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 11:08:36,035 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:36,035 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 11:08:36,035 INFO SenderThread:1789646 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 11:08:36,036 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 12 +2025-01-01 11:08:36,036 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:36,036 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 11:08:36,036 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:36,037 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 11:08:36,037 INFO SenderThread:1789646 [file_stream.py:finish():601] file stream finish called +2025-01-01 11:08:36,228 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:08:36,243 INFO SenderThread:1789646 [file_stream.py:finish():605] file stream finish is done +2025-01-01 11:08:36,243 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 13 +2025-01-01 11:08:36,243 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:08:36,243 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:36,243 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 11:08:36,244 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:36,244 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 11:08:36,244 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 14 +2025-01-01 11:08:36,244 DEBUG SenderThread:1789646 [sender.py:send():379] send: final +2025-01-01 11:08:36,244 DEBUG SenderThread:1789646 [sender.py:send():379] send: footer +2025-01-01 11:08:36,244 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:36,244 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 11:08:36,244 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:36,244 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 11:08:36,245 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:08:36,245 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:08:36,245 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 11:08:36,245 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:08:36,246 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 11:08:36,246 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: server_info +2025-01-01 11:08:36,247 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 11:08:36,248 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:36,296 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:08:36,296 INFO MainThread:1789646 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 11:08:36,296 INFO MainThread:1789646 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 11:08:36,297 INFO MainThread:1789646 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 11:08:36,297 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 11:08:36,297 INFO HandlerThread:1789646 [handler.py:finish():882] shutting down handler +2025-01-01 11:08:37,246 INFO WriterThread:1789646 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/run-hex7gk4r.wandb +2025-01-01 11:08:37,296 INFO SenderThread:1789646 [sender.py:finish():1615] shutting down sender +2025-01-01 11:08:37,296 INFO SenderThread:1789646 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:08:37,296 INFO SenderThread:1789646 [file_pusher.py:join():175] waiting for file pusher diff --git a/hex7gk4r/wandb/debug.log b/hex7gk4r/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..77e4fa2a2d958032d03b82dab85292d62c1d92c1 --- /dev/null +++ b/hex7gk4r/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_setup.py:_flush():76] Configure stats pid to 1789455 +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/settings +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/logs/debug.log +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/logs/debug-internal.log +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_init.py:init():569] calling init triggers +2025-01-01 11:03:03,225 INFO MainThread:1789455 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r'} +2025-01-01 11:03:03,225 INFO MainThread:1789455 [wandb_init.py:init():619] starting backend +2025-01-01 11:03:03,225 INFO MainThread:1789455 [wandb_init.py:init():623] setting up manager +2025-01-01 11:03:03,229 INFO MainThread:1789455 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 11:03:03,230 INFO MainThread:1789455 [wandb_init.py:init():631] backend started and connected +2025-01-01 11:03:03,245 INFO MainThread:1789455 [wandb_init.py:init():720] updated telemetry +2025-01-01 11:03:03,253 INFO MainThread:1789455 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 11:03:03,501 INFO MainThread:1789455 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 11:03:03,565 INFO MainThread:1789455 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 11:03:03,565 INFO MainThread:1789455 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 11:03:03,901 INFO MainThread:1789455 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 11:03:03,901 INFO MainThread:1789455 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 11:03:03,901 INFO MainThread:1789455 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 11:03:03,901 INFO MainThread:1789455 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 11:03:03,902 INFO MainThread:1789455 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 11:03:03,903 INFO MainThread:1789455 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/tb, True +2025-01-01 11:03:27,698 INFO MainThread:1789455 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-01 11:08:37,298 WARNING MsgRouterThr:1789455 [router.py:message_loop():77] message_loop has been closed diff --git a/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/code/FISH/eval_robot.py b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/config.yaml b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9512d56bd6706c47b7f9964b547f992bbdb8d46e --- /dev/null +++ b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1006_0' +wandb_notes: + desc: null + value: 1006_0_req_1352_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735747383 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/diff.patch b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..625cf877496560323ca22cd6b0a075ba511a2dc6 --- /dev/null +++ b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/diff.patch @@ -0,0 +1,106 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..01bdd86 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1007_0' ++# wandb_run_id: '1002_0' + ++# all books, 6/20 demos per book ++# RGBD+mask+act history ++# wandb_run_id: '1007_0' ++ ++# RGBD+mask+contact(all ftrs)+act history ++wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..50493fe 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -490,7 +490,8 @@ class Workspace: + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] +- total_num_episodes += filtered_run.summary_metrics['episode'] ++ # total_num_episodes += filtered_run.summary_metrics['episode'] ++ total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes +@@ -513,7 +514,6 @@ class Workspace: + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + +- + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index e3031af..6f97e26 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,11 +330,11 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.01 +- self.move_to_rot_tolerance = 2.0 ++ self.move_to_trans_tolerance = 0.005 ++ self.move_to_rot_tolerance = 1.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) +- self.move_to_target_timeout = 15.0 ++ self.move_to_target_timeout = 5.0 + self.move_to_target_max_rotation = 25.0 + # only can use one of these at a time + # assert use_roll + use_pitch + use_yaw <= 1, "Can only use one of roll, pitch, yaw" diff --git a/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..9239b8930ef50a5961d10da1c1fa840ac6307567 --- /dev/null +++ b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[20, 35, 0.5714285714285714, 0.3716529608029522, 0.7538510262458453, 0.410685271042567, 0.7356967329507549, 0.4360231012233896, 0.6986087858279691, 0.4541952274831351, 0.6842993311215397]]} \ No newline at end of file diff --git a/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/0_eval_0_0a25a53b5c8820b669aa.mp4 b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/0_eval_0_0a25a53b5c8820b669aa.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..434cba8df6afaa36901fc899837c2bec9c02756a --- /dev/null +++ b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/0_eval_0_0a25a53b5c8820b669aa.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a25a53b5c8820b669aa14051380155d22a72decf1d1e0caeb8374cbec73635b +size 1476161 diff --git a/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/1_eval_1_fae1d5a4084c3fa7e4ce.mp4 b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/1_eval_1_fae1d5a4084c3fa7e4ce.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a701bae63751fe7d2a01a649654639170a0b422f --- /dev/null +++ b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/1_eval_1_fae1d5a4084c3fa7e4ce.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fae1d5a4084c3fa7e4ce9bcd488a835aef4c52ce2bc8f173c5556165eb635f28 +size 1463141 diff --git a/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/2_eval_2_07f9274f756bd81cbaee.mp4 b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/2_eval_2_07f9274f756bd81cbaee.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ddfd806dd7114c9d6c34daf99d5dddea5f1e02b8 --- /dev/null +++ b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/2_eval_2_07f9274f756bd81cbaee.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07f9274f756bd81cbaeea939af0ed857deb6ec61b615f296fd45aa9ca85a9ed0 +size 1471142 diff --git a/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/3_eval_3_26ea14ca5e0897c83196.mp4 b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/3_eval_3_26ea14ca5e0897c83196.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..58d7a1a8c554017dbdca0d81a0271cfbb0ce3403 --- /dev/null +++ b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/3_eval_3_26ea14ca5e0897c83196.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26ea14ca5e0897c83196858a6de03ff39a6472b34e7de7ed4e68df94cb194a95 +size 1458359 diff --git a/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/4_eval_4_20669f7dea9bec0e6b76.mp4 b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/4_eval_4_20669f7dea9bec0e6b76.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ef64b79cd119d10ded6ea4a77d94991ceb4ff686 --- /dev/null +++ b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/4_eval_4_20669f7dea9bec0e6b76.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20669f7dea9bec0e6b764682fffadaf458c7316efe766c00e0e5648e29d8046c +size 1487542 diff --git a/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..722cb6e85dc07eb65e1b5f310179d2830b8e0564 --- /dev/null +++ b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log @@ -0,0 +1,279 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735747407.699615]: resetting environment +[INFO] [1735747407.704600]: cleared current plan +[INFO] [1735747407.705491]: moving to home +[INFO] [1735747408.707169]: reached home +[INFO] [1735747408.708499]: reset action history +[INFO] [1735747411.418739]: environment reset +Starting episode 0 +[INFO] [1735747411.424412]: resetting environment +[INFO] [1735747411.430649]: cleared current plan +[INFO] [1735747411.431692]: moving to home +[INFO] [1735747412.434057]: reached home +[INFO] [1735747412.435183]: reset action history +[INFO] [1735747415.145880]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735747429.602148]: resetting environment +[INFO] [1735747429.608278]: cleared current plan +[INFO] [1735747429.609281]: moving to home +[INFO] [1735747430.611521]: reached home +[INFO] [1735747430.612844]: reset action history +[INFO] [1735747433.323898]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/episode_rosbags/episode_0_2025-01-01-11-03-53.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1789455.log +started roslaunch server http://158.130.50.37:41665/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1789779] +process[print_text-2]: started with pid [1789780] +process[rosbag_record-3]: started with pid [1789781] +started bagging! +For topic gripper_width: timestamp difference is 110030161 for nearest: 1735747457745304571 - target: 1735747457635274410 at idx 249 +[INFO] [1735747468.404168]: Storing episode... +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[print_text-2] killing on exit +[INFO] [1735747469.178110]: Stored episode 1. +[INFO] [1735747469.180625]: Saving video... +[INFO] [1735747469.366141]: Video saved! +Starting episode 1 +[INFO] [1735747469.386178]: resetting environment +[INFO] [1735747469.398440]: cleared current plan +[INFO] [1735747469.399187]: moving to home +[INFO] [1735747473.801275]: reached home +[INFO] [1735747473.801779]: reset action history +[INFO] [1735747476.516610]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735747490.917799]: resetting environment +[INFO] [1735747490.921493]: cleared current plan +[INFO] [1735747490.921659]: moving to home +[INFO] [1735747491.923171]: reached home +[INFO] [1735747491.923665]: reset action history +[INFO] [1735747494.632200]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/episode_rosbags/episode_1_2025-01-01-11-04-54.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1789455.log +started roslaunch server http://158.130.50.37:33417/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1789907] +process[print_text-5]: started with pid [1789908] +process[rosbag_record-6]: started with pid [1789909] +started bagging! +[INFO] [1735747529.377778]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1735747530.177570]: Stored episode 2. +[INFO] [1735747530.177813]: Saving video... +[INFO] [1735747530.781024]: Video saved! +Starting episode 2 +[INFO] [1735747530.827313]: resetting environment +[INFO] [1735747530.840031]: cleared current plan +[INFO] [1735747530.840262]: moving to home +[WARN] [1735747530.745968]: Plan exhausted +[WARN] [1735747530.787877]: Plan exhausted +[WARN] [1735747530.824058]: Plan exhausted +[INFO] [1735747535.347815]: reached home +[INFO] [1735747535.350960]: reset action history +[INFO] [1735747538.130284]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735747540.429176]: resetting environment +[INFO] [1735747540.434774]: cleared current plan +[INFO] [1735747540.434969]: moving to home +[INFO] [1735747541.436514]: reached home +[INFO] [1735747541.437027]: reset action history +[INFO] [1735747544.145019]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/episode_rosbags/episode_2_2025-01-01-11-05-44.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1789455.log +started roslaunch server http://158.130.50.37:46843/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1790052] +process[print_text-8]: started with pid [1790053] +process[rosbag_record-9]: started with pid [1790054] +started bagging! +For topic gripper_width: timestamp difference is -110099005 for nearest: 1735747556378641201 - target: 1735747556488740206 at idx 167 +For topic gripper_width: timestamp difference is -117909437 for nearest: 1735747557178638214 - target: 1735747557296547651 at idx 174 +For topic gripper_width: timestamp difference is -119552545 for nearest: 1735747558478638001 - target: 1735747558598190546 at idx 188 +[INFO] [1735747578.933429]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1735747579.882523]: Stored episode 3. +[INFO] [1735747579.891616]: Saving video... +[INFO] [1735747580.397512]: Video saved! +Starting episode 3 +[INFO] [1735747580.460255]: resetting environment +[INFO] [1735747580.475728]: cleared current plan +[INFO] [1735747580.476093]: moving to home +[WARN] [1735747580.308223]: Plan exhausted +[WARN] [1735747580.348660]: Plan exhausted +[WARN] [1735747580.387085]: Plan exhausted +[WARN] [1735747580.427516]: Plan exhausted +[WARN] [1735747580.467514]: Plan exhausted +[INFO] [1735747585.082679]: reached home +[INFO] [1735747585.085899]: reset action history +[INFO] [1735747587.880913]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735747589.976891]: resetting environment +[INFO] [1735747589.997545]: cleared current plan +[INFO] [1735747589.999438]: moving to home +[INFO] [1735747591.010190]: reached home +[INFO] [1735747591.016242]: reset action history +[INFO] [1735747593.794560]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/episode_rosbags/episode_3_2025-01-01-11-06-33.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1789455.log +started roslaunch server http://158.130.50.37:35483/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1790194] +process[print_text-11]: started with pid [1790195] +process[rosbag_record-12]: started with pid [1790196] +started bagging! +For topic gripper_width: timestamp difference is 108767146 for nearest: 1735747614278688306 - target: 1735747614169921160 at idx 249 +For topic gripper_width: timestamp difference is 109307609 for nearest: 1735747614545328937 - target: 1735747614436021328 at idx 249 +For topic gripper_width: timestamp difference is 107821809 for nearest: 1735747620045262681 - target: 1735747619937440872 at idx 249 +[INFO] [1735747628.613479]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1735747629.386262]: Stored episode 4. +[INFO] [1735747629.387041]: Saving video... +[INFO] [1735747629.652381]: Video saved! +Starting episode 4 +[INFO] [1735747629.710719]: resetting environment +[INFO] [1735747629.732976]: cleared current plan +[INFO] [1735747629.733219]: moving to home +[INFO] [1735747634.141052]: reached home +[INFO] [1735747634.146775]: reset action history +[INFO] [1735747636.963061]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735747643.868106]: resetting environment +[INFO] [1735747643.901444]: cleared current plan +[INFO] [1735747643.901977]: moving to home +Move to target timeout: 5.003791809082031. Trans error: 0.005061346701185133. Rot error: 0.9187980160386432 +[INFO] [1735747649.911289]: reached home +[INFO] [1735747649.918016]: reset action history +[INFO] [1735747652.715253]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/episode_rosbags/episode_4_2025-01-01-11-07-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1789455.log +started roslaunch server http://158.130.50.37:34593/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1790335] +process[print_text-14]: started with pid [1790336] +process[rosbag_record-15]: started with pid [1790360] +started bagging! +For topic gripper_width: timestamp difference is 107290808 for nearest: 1735747655278666559 - target: 1735747655171375751 at idx 35 +For topic gripper_width: timestamp difference is 110415879 for nearest: 1735747656845303956 - target: 1735747656734888077 at idx 55 +[INFO] [1735747687.564911]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735747688.405678]: Stored episode 5. +[INFO] [1735747688.408978]: Saving video... +[INFO] [1735747688.788721]: Video saved! +[INFO] [1735747688.841939]: resetting environment +[INFO] [1735747688.881866]: cleared current plan +[INFO] [1735747688.885421]: moving to home +[INFO] [1735747693.514199]: reached home +[INFO] [1735747693.518535]: reset action history +[INFO] [1735747696.345061]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/requirements.txt b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-metadata.json b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..8c00452a3af578574084b0bdeb9b76f4bd0c56ea --- /dev/null +++ b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T16:03:03.590194", + "startedAt": "2025-01-01T16:03:03.217444", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3561.171875, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.942, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 664.0837097167969 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..10668612c585a043b79e10000b3a8c8c32a40505 --- /dev/null +++ b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "0a25a53b5c8820b669aa14051380155d22a72decf1d1e0caeb8374cbec73635b", "size": 1476161, "path": "media/videos/eval/0_eval_0_0a25a53b5c8820b669aa.mp4"}, "global_step": 665, "_timestamp": 1735747704.6028068, "_runtime": 321.37183475494385, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "fae1d5a4084c3fa7e4ce9bcd488a835aef4c52ce2bc8f173c5556165eb635f28", "size": 1463141, "path": "media/videos/eval/1_eval_1_fae1d5a4084c3fa7e4ce.mp4"}, "eval/num_success": 5.0, "episode": 35.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "07f9274f756bd81cbaeea939af0ed857deb6ec61b615f296fd45aa9ca85a9ed0", "size": 1471142, "path": "media/videos/eval/2_eval_2_07f9274f756bd81cbaee.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "26ea14ca5e0897c83196858a6de03ff39a6472b34e7de7ed4e68df94cb194a95", "size": 1458359, "path": "media/videos/eval/3_eval_3_26ea14ca5e0897c83196.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "20669f7dea9bec0e6b764682fffadaf458c7316efe766c00e0e5648e29d8046c", "size": 1487542, "path": "media/videos/eval/4_eval_4_20669f7dea9bec0e6b76.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://s79ib0iiqo344b8me9k0g0sll55d5lmzw8o3h0szgkmwols4s0fsnt20p00en4j01idlvlmb5vakc4ipbk8i3x8enot36dbp88s776ewfcym9ihbvlps8mjrxrf8tfqz/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://7vi1w2748ky59gvn2h2dvde7q6pihx7leoylxfpq9loxxkmi3fhflgxv9zg3v73uj38k9bar8bfx2fdja5pwveqouoezvyi2rh9y9t1rltfvb4jzehyo6avlcf7dekzc:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 20, "total_num_episodes": 35, "total_success_rate": 0.5714285714285714, "total_success_rate_lb_0.01": 0.3716529608029522, "total_success_rate_ub_0.01": 0.7538510262458453, "total_success_rate_lb_0.025": 0.410685271042567, "total_success_rate_ub_0.025": 0.7356967329507549, "total_success_rate_lb_0.05": 0.4360231012233896, "total_success_rate_ub_0.05": 0.6986087858279691, "total_success_rate_lb_0.1": 0.4541952274831351, "total_success_rate_ub_0.1": 0.6842993311215397, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "51a31336f9897b8b206f5f40b68b24414baba81b9933e86c5999ebc818195da9", "size": 519, "artifact_path": "wandb-client-artifact://pajkqiwhh5ykda2q2ebry1xkbrqvt62p7kcazpgosfkl623f9na6lw3115u8tuml9og6veq0341y1l21svmuun3tfwhu68db1qji5qovl4n2w2surmuv52vwiv0n4alr/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://3zb9lp6l8gf247kxgv54wo9a5wj8k9pnuv3fy35cy673p3bylc4x2j1m7tqpw1sv2wo2hu3dyc1hgejtu3q16ixsynrwz80iid4yuas7cannjss2uratwd4hs7zqhlkp:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 321}, "eval/total_success_rate": 0.5714285969734192, "eval/total_success_rate_lb_0.01": 0.3716529607772827, "eval/total_success_rate_ub_0.01": 0.7538509964942932, "eval/total_success_rate_lb_0.025": 0.410685271024704, "eval/total_success_rate_ub_0.025": 0.7356967329978943, "eval/total_success_rate_lb_0.05": 0.43602311611175537, "eval/total_success_rate_ub_0.05": 0.6986087560653687, "eval/total_success_rate_lb_0.1": 0.4541952311992645, "eval/total_success_rate_ub_0.1": 0.6842993497848511} \ No newline at end of file diff --git a/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/logs/debug-internal.log b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7ef422b0ddd046526174c27432863f831939048c --- /dev/null +++ b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/logs/debug-internal.log @@ -0,0 +1,845 @@ +2025-01-01 11:03:03,231 INFO StreamThr :1789646 [internal.py:wandb_internal():85] W&B internal server running at pid: 1789646, started at: 2025-01-01 11:03:03.230653 +2025-01-01 11:03:03,232 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status +2025-01-01 11:03:03,245 INFO WriterThread:1789646 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/run-hex7gk4r.wandb +2025-01-01 11:03:03,246 DEBUG SenderThread:1789646 [sender.py:send():379] send: header +2025-01-01 11:03:03,256 DEBUG SenderThread:1789646 [sender.py:send():379] send: run +2025-01-01 11:03:03,488 INFO SenderThread:1789646 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files +2025-01-01 11:03:03,488 INFO SenderThread:1789646 [sender.py:_start_run_threads():1188] run started: hex7gk4r with start time 1735747383.230972 +2025-01-01 11:03:03,501 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 11:03:03,501 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: check_version +2025-01-01 11:03:03,570 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 11:03:03,575 DEBUG HandlerThread:1789646 [system_info.py:__init__():26] System info init +2025-01-01 11:03:03,575 DEBUG HandlerThread:1789646 [system_info.py:__init__():41] System info init done +2025-01-01 11:03:03,575 INFO HandlerThread:1789646 [system_monitor.py:start():194] Starting system monitor +2025-01-01 11:03:03,576 INFO SystemMonitor:1789646 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 11:03:03,576 INFO HandlerThread:1789646 [system_monitor.py:probe():214] Collecting system info +2025-01-01 11:03:03,577 INFO SystemMonitor:1789646 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 11:03:03,577 INFO SystemMonitor:1789646 [interfaces.py:start():188] Started disk monitoring +2025-01-01 11:03:03,578 INFO SystemMonitor:1789646 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 11:03:03,578 INFO SystemMonitor:1789646 [interfaces.py:start():188] Started memory monitoring +2025-01-01 11:03:03,579 INFO SystemMonitor:1789646 [interfaces.py:start():188] Started network monitoring +2025-01-01 11:03:03,590 DEBUG HandlerThread:1789646 [system_info.py:probe():152] Probing system +2025-01-01 11:03:03,591 DEBUG HandlerThread:1789646 [system_info.py:_probe_git():137] Probing git +2025-01-01 11:03:03,596 DEBUG HandlerThread:1789646 [system_info.py:_probe_git():145] Probing git done +2025-01-01 11:03:03,596 DEBUG HandlerThread:1789646 [system_info.py:probe():200] Probing system done +2025-01-01 11:03:03,596 DEBUG HandlerThread:1789646 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T16:03:03.590194', 'startedAt': '2025-01-01T16:03:03.217444', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.171875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.942, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 664.0837097167969}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 11:03:03,596 INFO HandlerThread:1789646 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 11:03:03,596 INFO HandlerThread:1789646 [system_monitor.py:probe():227] Publishing system info +2025-01-01 11:03:03,596 DEBUG HandlerThread:1789646 [system_info.py:_save_code():44] Saving code +2025-01-01 11:03:03,598 DEBUG HandlerThread:1789646 [system_info.py:_save_code():67] Saving code done +2025-01-01 11:03:03,598 DEBUG HandlerThread:1789646 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 11:03:03,659 DEBUG HandlerThread:1789646 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 11:03:03,659 INFO HandlerThread:1789646 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 11:03:03,662 DEBUG SenderThread:1789646 [sender.py:send():379] send: files +2025-01-01 11:03:03,662 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 11:03:03,663 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 11:03:03,663 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 11:03:03,901 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 11:03:03,901 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: python_packages +2025-01-01 11:03:03,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:03:03,902 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:03:03,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:03,903 INFO HandlerThread:1789646 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/tb" + save: true +} +control { + relay_id: "9383c060aecf4fb1bae46e64e533ff79" +} +_info { + stream_id: "hex7gk4r" +} + +2025-01-01 11:03:03,904 DEBUG HandlerThread:1789646 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 11:03:03,911 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/m540yw5f-wandb-metadata.json +2025-01-01 11:03:03,917 INFO wandb-upload_1:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/5hwe7lky-code/FISH/eval_robot.py +2025-01-01 11:03:03,917 INFO wandb-upload_2:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/r9xivxet-diff.patch +2025-01-01 11:03:04,005 DEBUG SenderThread:1789646 [sender.py:send():379] send: telemetry +2025-01-01 11:03:04,005 DEBUG SenderThread:1789646 [sender.py:send():379] send: tbrecord +2025-01-01 11:03:04,005 DEBUG SenderThread:1789646 [sender.py:send():379] send: files +2025-01-01 11:03:04,005 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 with policy live +2025-01-01 11:03:04,206 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/el9lws25-events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:03:04,489 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/requirements.txt +2025-01-01 11:03:04,490 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:03:04,490 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:03:04,490 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/diff.patch +2025-01-01 11:03:04,490 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/code/FISH/eval_robot.py +2025-01-01 11:03:04,490 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-metadata.json +2025-01-01 11:03:04,490 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/code/FISH +2025-01-01 11:03:04,490 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/code +2025-01-01 11:03:04,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:05,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:06,489 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:03:06,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:07,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:08,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:08,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:09,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:10,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:11,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:12,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:13,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:13,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:14,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:15,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:16,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:17,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:18,901 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:03:18,901 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:03:18,945 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:18,960 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:19,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:20,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:21,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:22,904 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:23,904 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:24,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:24,904 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:25,904 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:26,904 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:27,698 DEBUG SenderThread:1789646 [sender.py:send():379] send: config +2025-01-01 11:03:27,904 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:28,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:29,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:29,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:30,495 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:03:30,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:31,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:32,496 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:03:32,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:33,901 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:03:33,902 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:03:33,945 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:34,496 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:03:34,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:34,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:35,496 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/config.yaml +2025-01-01 11:03:35,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:36,497 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:03:36,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:37,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:38,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:39,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:40,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:40,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:41,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:42,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:43,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:44,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:45,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:45,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:46,908 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:47,907 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:48,901 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:03:48,902 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:03:48,945 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:49,907 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:50,908 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:51,653 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:51,908 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:52,501 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:03:52,908 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:53,908 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:54,502 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:03:54,908 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:55,909 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:56,502 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:03:56,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:03:56,911 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:57,910 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:58,910 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:03:59,910 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:00,910 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:01,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:01,910 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:02,911 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:03,579 DEBUG SystemMonitor:1789646 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 11:04:03,579 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:04:03,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:04:03,902 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:04:03,945 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:04,911 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:05,911 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:06,911 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:07,003 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:07,911 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:08,911 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:09,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:10,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:11,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:12,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:12,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:13,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:14,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:15,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:16,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:17,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:18,790 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:18,902 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:04:18,902 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:04:18,945 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:19,912 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:20,508 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:04:20,913 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:21,913 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:22,913 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:23,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:23,913 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:24,913 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:25,913 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:26,913 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:27,913 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:28,910 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:28,914 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:29,382 DEBUG SenderThread:1789646 [sender.py:send():379] send: files +2025-01-01 11:04:29,382 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:04:29,382 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_0a25a53b5c8820b669aa.mp4 with policy now +2025-01-01 11:04:29,383 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:04:29,383 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:04:29,384 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:04:29,511 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/0_eval_0_0a25a53b5c8820b669aa.mp4 +2025-01-01 11:04:29,511 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:04:29,511 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media +2025-01-01 11:04:29,511 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval +2025-01-01 11:04:29,511 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos +2025-01-01 11:04:29,653 INFO wandb-upload_1:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/ob5kp906-media/videos/eval/0_eval_0_0a25a53b5c8820b669aa.mp4 +2025-01-01 11:04:29,914 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:29,951 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/72hg7fzg-media/videos/eval/0_eval_0_0a25a53b5c8820b669aa.mp4 +2025-01-01 11:04:30,511 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:04:30,914 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:31,915 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:32,915 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:33,580 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:04:33,904 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:04:33,904 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:04:33,915 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:34,005 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:34,915 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:35,919 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:36,513 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:04:36,915 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:37,915 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:38,514 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:04:38,915 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:39,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:39,916 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:40,916 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:41,916 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:42,916 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:43,916 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:44,916 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:45,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:45,916 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:46,916 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:47,917 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:48,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:04:48,903 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:04:48,949 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:49,917 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:50,913 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:50,961 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:51,518 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:04:51,759 INFO wandb-upload_1:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/l69935fl-events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:04:51,917 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:52,518 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:04:52,917 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:53,917 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:54,519 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:04:54,918 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:55,918 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:56,520 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:04:56,886 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:04:56,918 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:57,919 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:58,919 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:04:59,919 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:00,919 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:01,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:01,919 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:02,919 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:03,580 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:05:03,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:05:03,903 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:05:03,949 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:04,919 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:05,919 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:06,920 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:06,987 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:07,920 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:08,920 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:09,920 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:10,920 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:11,920 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:12,907 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:12,920 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:13,920 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:14,920 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:15,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:16,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:17,907 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:17,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:18,903 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:05:18,903 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:05:18,945 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:19,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:20,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:21,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:22,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:23,030 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:23,922 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:24,922 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:25,922 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:26,922 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:27,922 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:28,907 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:28,922 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:29,923 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:30,531 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:05:30,819 DEBUG SenderThread:1789646 [sender.py:send():379] send: files +2025-01-01 11:05:30,820 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_fae1d5a4084c3fa7e4ce.mp4 with policy now +2025-01-01 11:05:30,820 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:05:30,821 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:05:30,821 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:05:30,821 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:05:30,923 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:31,081 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/aq2sgdsa-media/videos/eval/1_eval_1_fae1d5a4084c3fa7e4ce.mp4 +2025-01-01 11:05:31,531 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:05:31,532 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/1_eval_1_fae1d5a4084c3fa7e4ce.mp4 +2025-01-01 11:05:31,532 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval +2025-01-01 11:05:31,807 INFO wandb-upload_1:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/3wtb49tu-media/videos/eval/1_eval_1_fae1d5a4084c3fa7e4ce.mp4 +2025-01-01 11:05:31,923 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:32,532 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:05:32,925 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:33,532 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:05:33,581 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:05:33,909 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:05:33,910 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:05:33,953 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:34,067 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:34,927 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:35,925 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:36,533 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:05:36,930 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:37,925 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:38,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:39,908 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:39,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:40,534 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:05:40,535 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:05:40,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:41,474 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:05:41,474 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:05:41,474 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:05:41,535 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:05:41,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:42,535 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:05:42,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:43,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:44,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:45,395 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:45,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:46,536 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:05:46,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:47,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:48,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:05:48,905 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:05:48,949 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:49,926 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:50,927 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:50,965 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:51,927 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:52,927 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:53,927 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:54,927 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:55,927 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:56,578 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:05:56,927 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:57,927 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:58,540 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:05:58,931 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:05:59,928 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:00,541 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:06:00,928 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:01,910 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:01,928 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:02,928 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:03,581 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:06:03,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:06:03,905 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:06:03,949 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:04,929 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:05,929 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:06,929 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:07,022 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:07,929 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:08,929 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:09,929 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:10,930 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:11,930 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:12,910 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:12,930 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:13,930 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:14,930 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:15,930 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:16,930 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:17,911 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:17,930 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:18,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:06:18,905 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:06:18,953 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:19,933 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:20,454 DEBUG SenderThread:1789646 [sender.py:send():379] send: files +2025-01-01 11:06:20,454 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_07f9274f756bd81cbaee.mp4 with policy now +2025-01-01 11:06:20,455 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:06:20,455 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:06:20,456 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:06:20,471 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:06:20,548 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:06:20,548 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:06:20,548 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/2_eval_2_07f9274f756bd81cbaee.mp4 +2025-01-01 11:06:20,548 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval +2025-01-01 11:06:20,708 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/okcuosdj-media/videos/eval/2_eval_2_07f9274f756bd81cbaee.mp4 +2025-01-01 11:06:20,931 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:20,965 INFO wandb-upload_1:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/lfz8ukdf-media/videos/eval/2_eval_2_07f9274f756bd81cbaee.mp4 +2025-01-01 11:06:21,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:22,548 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:06:22,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:23,476 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:23,549 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:06:23,936 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:24,942 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:25,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:26,550 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:06:26,940 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:27,937 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:28,918 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:28,938 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:29,942 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:30,235 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:06:30,235 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:06:30,235 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:06:30,551 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:06:30,551 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:06:30,551 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:06:30,785 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/0kxa3lse-events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:06:30,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:31,940 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:32,552 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:06:32,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:33,582 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:06:33,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:06:33,905 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:06:33,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:33,995 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:34,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:35,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:36,553 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:06:36,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:37,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:38,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:39,915 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:39,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:40,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:41,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:42,941 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:43,942 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:44,915 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:44,942 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:45,942 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:46,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:47,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:48,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:06:48,905 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:06:48,949 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:49,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:50,013 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:50,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:51,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:52,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:53,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:54,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:55,550 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:06:55,943 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:56,560 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:06:56,944 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:57,944 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:58,944 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:06:59,944 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:00,944 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:01,093 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:01,944 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:02,562 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:02,944 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:03,582 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:07:03,905 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:07:03,906 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:07:03,949 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:04,944 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:05,944 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:06,946 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:07,052 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:07,946 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:08,949 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:09,684 DEBUG SenderThread:1789646 [sender.py:send():379] send: files +2025-01-01 11:07:09,684 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_26ea14ca5e0897c83196.mp4 with policy now +2025-01-01 11:07:09,687 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:07:09,688 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:07:09,688 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:07:09,734 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:07:09,933 INFO wandb-upload_1:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/m2mc05m1-media/videos/eval/3_eval_3_26ea14ca5e0897c83196.mp4 +2025-01-01 11:07:09,949 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:10,565 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:10,565 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:07:10,565 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/3_eval_3_26ea14ca5e0897c83196.mp4 +2025-01-01 11:07:10,565 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval +2025-01-01 11:07:10,876 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/uxyl7jzq-media/videos/eval/3_eval_3_26ea14ca5e0897c83196.mp4 +2025-01-01 11:07:10,950 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:11,953 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:12,735 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:12,953 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:13,960 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:14,962 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:15,957 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:16,567 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:16,960 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:17,964 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:18,009 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:18,567 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:18,917 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:07:18,917 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:07:18,962 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:19,966 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:20,960 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:21,960 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:22,961 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:23,841 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:23,956 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:24,302 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:07:24,302 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:07:24,302 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:07:24,569 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:07:24,569 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:07:24,964 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:25,969 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:26,570 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:26,971 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:27,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:28,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:28,963 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:29,975 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:30,572 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:30,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:31,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:32,572 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:32,966 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:33,583 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:07:33,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:07:33,906 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:07:33,966 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:34,015 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:34,574 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:34,966 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:35,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:36,574 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:36,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:37,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:38,575 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:07:38,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:39,922 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:39,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:40,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:41,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:42,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:43,967 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:44,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:45,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:45,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:46,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:47,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:48,906 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:07:48,906 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:07:48,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:49,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:50,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:50,969 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:51,968 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:52,969 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:53,971 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:54,971 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:55,971 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:56,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:07:56,972 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:57,971 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:58,972 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:07:59,972 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:00,972 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:01,972 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:02,921 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:08:02,972 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:03,583 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:08:03,907 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:08:03,907 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:08:03,972 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:04,973 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:05,973 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:06,973 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:07,977 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:08,407 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:08:08,587 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:08:08,839 DEBUG SenderThread:1789646 [sender.py:send():379] send: files +2025-01-01 11:08:08,839 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_20669f7dea9bec0e6b76.mp4 with policy now +2025-01-01 11:08:08,841 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:08:08,842 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:08:08,842 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:08,842 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:08,978 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:09,094 INFO wandb-upload_1:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/5gyuhmzl-media/videos/eval/4_eval_4_20669f7dea9bec0e6b76.mp4 +2025-01-01 11:08:09,587 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:08:09,587 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/4_eval_4_20669f7dea9bec0e6b76.mp4 +2025-01-01 11:08:09,588 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval +2025-01-01 11:08:09,869 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/2fj0xs0t-media/videos/eval/4_eval_4_20669f7dea9bec0e6b76.mp4 +2025-01-01 11:08:09,981 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:10,587 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:08:10,988 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:11,995 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:12,984 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:13,514 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:08:13,996 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:14,589 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:08:14,988 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:15,994 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:16,993 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:17,993 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:18,590 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:08:18,922 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:08:18,923 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:08:18,992 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:08:18,992 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:19,996 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:20,995 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:21,995 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:23,014 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:24,005 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:24,159 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:08:24,159 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:08:24,298 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:08:24,592 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:08:24,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:08:24,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:08:24,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media +2025-01-01 11:08:24,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table/eval +2025-01-01 11:08:24,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table +2025-01-01 11:08:24,650 INFO wandb-upload_1:1789646 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpnor_r9wg +2025-01-01 11:08:24,693 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:08:24,701 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:08:24,769 INFO wandb-upload_2:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/8rhbygf3-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:08:24,841 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/izvtcfdg-events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:08:24,998 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:25,052 INFO SenderThread:1789646 [sender.py:send_request_log_artifact():1525] logged artifact run-hex7gk4r-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY3NDAzMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MDIwNg==', 'latestArtifact': None}} +2025-01-01 11:08:25,052 DEBUG SenderThread:1789646 [sender.py:send():379] send: files +2025-01-01 11:08:25,052 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-01 11:08:25,053 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:08:25,053 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:08:25,053 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,054 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,055 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:08:25,055 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,058 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,059 DEBUG SenderThread:1789646 [sender.py:send():379] send: telemetry +2025-01-01 11:08:25,059 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,059 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,059 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,059 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,060 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,060 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,060 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,060 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,060 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,061 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,061 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,061 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,061 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,062 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,062 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,062 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,062 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,063 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,063 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,063 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,063 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,063 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,064 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:08:25,297 INFO wandb-upload_1:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/8eoze2vw-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:08:25,437 INFO wandb-upload_0:1789646 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpz1gbqcms +2025-01-01 11:08:25,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:08:25,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:08:25,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json +2025-01-01 11:08:25,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table/eval +2025-01-01 11:08:25,804 INFO SenderThread:1789646 [sender.py:send_request_log_artifact():1525] logged artifact run-hex7gk4r-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY3NDA0Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MDIxMQ==', 'latestArtifact': None}} +2025-01-01 11:08:25,804 DEBUG SenderThread:1789646 [sender.py:send():379] send: files +2025-01-01 11:08:25,804 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json with policy now +2025-01-01 11:08:25,804 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:08:25,804 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:25,805 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,805 DEBUG SenderThread:1789646 [sender.py:send():379] send: exit +2025-01-01 11:08:25,805 INFO SenderThread:1789646 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 11:08:25,805 INFO SenderThread:1789646 [sender.py:send_exit():588] handling runtime: 321 +2025-01-01 11:08:25,806 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:25,806 INFO SenderThread:1789646 [sender.py:send_exit():594] send defer +2025-01-01 11:08:25,806 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:25,806 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 11:08:25,806 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:25,806 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 11:08:25,806 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 1 +2025-01-01 11:08:25,806 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:25,807 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 11:08:25,807 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:25,807 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 11:08:25,807 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 2 +2025-01-01 11:08:25,807 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:25,807 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 11:08:25,807 INFO HandlerThread:1789646 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 11:08:25,807 DEBUG SystemMonitor:1789646 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 11:08:25,807 INFO HandlerThread:1789646 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 11:08:25,807 DEBUG SystemMonitor:1789646 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 11:08:25,808 INFO HandlerThread:1789646 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 11:08:25,860 INFO HandlerThread:1789646 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 11:08:25,860 INFO HandlerThread:1789646 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 11:08:25,861 INFO HandlerThread:1789646 [interfaces.py:finish():200] Joined network monitor +2025-01-01 11:08:25,861 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:25,861 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 11:08:25,861 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 3 +2025-01-01 11:08:25,861 DEBUG SenderThread:1789646 [sender.py:send():379] send: stats +2025-01-01 11:08:25,861 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:25,861 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 11:08:25,862 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:25,862 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 11:08:25,862 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 4 +2025-01-01 11:08:25,862 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:25,862 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 11:08:25,976 INFO wandb-upload_2:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/fwtuz5zp-media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json +2025-01-01 11:08:26,223 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/6n7joruu-media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json +2025-01-01 11:08:26,593 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:08:26,594 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:08:31,595 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/config.yaml +2025-01-01 11:08:32,380 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:08:32,380 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:32,380 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 11:08:32,380 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:08:32,380 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 5 +2025-01-01 11:08:32,380 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 11:08:32,380 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:08:32,381 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:32,382 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 11:08:32,382 DEBUG SenderThread:1789646 [sender.py:send():379] send: history +2025-01-01 11:08:32,382 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:08:32,383 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:32,383 DEBUG SenderThread:1789646 [sender.py:send():379] send: summary +2025-01-01 11:08:32,387 INFO SenderThread:1789646 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:08:32,387 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:32,387 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 11:08:32,387 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 6 +2025-01-01 11:08:32,387 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:32,387 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 11:08:32,387 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:32,387 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 11:08:32,388 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 7 +2025-01-01 11:08:32,388 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:08:32,388 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:32,388 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 11:08:32,388 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:32,388 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 11:08:32,596 INFO Thread-12 :1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:08:33,227 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:08:34,935 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 8 +2025-01-01 11:08:34,935 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:08:34,936 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:34,936 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 11:08:34,936 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:34,936 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 11:08:34,936 INFO SenderThread:1789646 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 11:08:34,938 INFO SenderThread:1789646 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 11:08:34,940 INFO SenderThread:1789646 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 11:08:34,949 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 9 +2025-01-01 11:08:34,950 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:34,950 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 11:08:34,950 DEBUG SenderThread:1789646 [sender.py:send():379] send: artifact +2025-01-01 11:08:35,133 INFO SenderThread:1789646 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIyMjk2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 11:08:35,133 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:35,133 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 11:08:35,133 INFO SenderThread:1789646 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 11:08:35,227 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:08:35,597 INFO SenderThread:1789646 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:08:35,597 INFO SenderThread:1789646 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files +2025-01-01 11:08:35,597 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/config.yaml config.yaml +2025-01-01 11:08:35,597 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json wandb-summary.json +2025-01-01 11:08:35,597 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:08:35,598 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/requirements.txt requirements.txt +2025-01-01 11:08:35,598 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-metadata.json wandb-metadata.json +2025-01-01 11:08:35,598 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log output.log +2025-01-01 11:08:35,600 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/diff.patch diff.patch +2025-01-01 11:08:35,602 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 11:08:35,606 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/3_eval_3_26ea14ca5e0897c83196.mp4 media/videos/eval/3_eval_3_26ea14ca5e0897c83196.mp4 +2025-01-01 11:08:35,606 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/4_eval_4_20669f7dea9bec0e6b76.mp4 media/videos/eval/4_eval_4_20669f7dea9bec0e6b76.mp4 +2025-01-01 11:08:35,606 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/1_eval_1_fae1d5a4084c3fa7e4ce.mp4 media/videos/eval/1_eval_1_fae1d5a4084c3fa7e4ce.mp4 +2025-01-01 11:08:35,606 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/0_eval_0_0a25a53b5c8820b669aa.mp4 media/videos/eval/0_eval_0_0a25a53b5c8820b669aa.mp4 +2025-01-01 11:08:35,606 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/videos/eval/2_eval_2_07f9274f756bd81cbaee.mp4 media/videos/eval/2_eval_2_07f9274f756bd81cbaee.mp4 +2025-01-01 11:08:35,606 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json +2025-01-01 11:08:35,607 INFO SenderThread:1789646 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 11:08:35,607 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 10 +2025-01-01 11:08:35,607 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:08:35,607 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:35,607 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 11:08:35,607 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:35,607 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 11:08:35,607 INFO SenderThread:1789646 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:08:35,776 INFO wandb-upload_1:1789646 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/wandb-summary.json +2025-01-01 11:08:35,781 INFO wandb-upload_2:1789646 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/config.yaml +2025-01-01 11:08:35,828 INFO wandb-upload_3:1789646 [upload_job.py:push():130] Uploaded file /tmp/tmpkhe9yb83wandb/lj4wjqap-events.out.tfevents.1735747383.leonmkim-ROG-Strix-G15CS-G15CS.1789455.0 +2025-01-01 11:08:35,833 INFO wandb-upload_4:1789646 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/output.log +2025-01-01 11:08:35,834 INFO wandb-upload_0:1789646 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/files/requirements.txt +2025-01-01 11:08:36,034 INFO Thread-11 (_thread_body):1789646 [sender.py:transition_state():614] send defer: 11 +2025-01-01 11:08:36,034 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:36,035 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 11:08:36,035 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:36,035 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 11:08:36,035 INFO SenderThread:1789646 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 11:08:36,036 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 12 +2025-01-01 11:08:36,036 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:36,036 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 11:08:36,036 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:36,037 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 11:08:36,037 INFO SenderThread:1789646 [file_stream.py:finish():601] file stream finish called +2025-01-01 11:08:36,228 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:08:36,243 INFO SenderThread:1789646 [file_stream.py:finish():605] file stream finish is done +2025-01-01 11:08:36,243 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 13 +2025-01-01 11:08:36,243 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:08:36,243 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:36,243 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 11:08:36,244 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:36,244 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 11:08:36,244 INFO SenderThread:1789646 [sender.py:transition_state():614] send defer: 14 +2025-01-01 11:08:36,244 DEBUG SenderThread:1789646 [sender.py:send():379] send: final +2025-01-01 11:08:36,244 DEBUG SenderThread:1789646 [sender.py:send():379] send: footer +2025-01-01 11:08:36,244 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:08:36,244 INFO HandlerThread:1789646 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 11:08:36,244 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: defer +2025-01-01 11:08:36,244 INFO SenderThread:1789646 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 11:08:36,245 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:08:36,245 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:08:36,245 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 11:08:36,245 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:08:36,246 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 11:08:36,246 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: server_info +2025-01-01 11:08:36,247 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 11:08:36,248 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:08:36,296 DEBUG SenderThread:1789646 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:08:36,296 INFO MainThread:1789646 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 11:08:36,296 INFO MainThread:1789646 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 11:08:36,297 INFO MainThread:1789646 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 11:08:36,297 DEBUG HandlerThread:1789646 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 11:08:36,297 INFO HandlerThread:1789646 [handler.py:finish():882] shutting down handler +2025-01-01 11:08:37,246 INFO WriterThread:1789646 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/run-hex7gk4r.wandb +2025-01-01 11:08:37,296 INFO SenderThread:1789646 [sender.py:finish():1615] shutting down sender +2025-01-01 11:08:37,296 INFO SenderThread:1789646 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:08:37,296 INFO SenderThread:1789646 [file_pusher.py:join():175] waiting for file pusher diff --git a/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/logs/debug.log b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..77e4fa2a2d958032d03b82dab85292d62c1d92c1 --- /dev/null +++ b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_setup.py:_flush():76] Configure stats pid to 1789455 +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/settings +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/logs/debug.log +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/logs/debug-internal.log +2025-01-01 11:03:03,224 INFO MainThread:1789455 [wandb_init.py:init():569] calling init triggers +2025-01-01 11:03:03,225 INFO MainThread:1789455 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r'} +2025-01-01 11:03:03,225 INFO MainThread:1789455 [wandb_init.py:init():619] starting backend +2025-01-01 11:03:03,225 INFO MainThread:1789455 [wandb_init.py:init():623] setting up manager +2025-01-01 11:03:03,229 INFO MainThread:1789455 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 11:03:03,230 INFO MainThread:1789455 [wandb_init.py:init():631] backend started and connected +2025-01-01 11:03:03,245 INFO MainThread:1789455 [wandb_init.py:init():720] updated telemetry +2025-01-01 11:03:03,253 INFO MainThread:1789455 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 11:03:03,501 INFO MainThread:1789455 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 11:03:03,565 INFO MainThread:1789455 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 11:03:03,565 INFO MainThread:1789455 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 11:03:03,901 INFO MainThread:1789455 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 11:03:03,901 INFO MainThread:1789455 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 11:03:03,901 INFO MainThread:1789455 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 11:03:03,901 INFO MainThread:1789455 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 11:03:03,902 INFO MainThread:1789455 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 11:03:03,903 INFO MainThread:1789455 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/hex7gk4r/tb, True +2025-01-01 11:03:27,698 INFO MainThread:1789455 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-01 11:08:37,298 WARNING MsgRouterThr:1789455 [router.py:message_loop():77] message_loop has been closed diff --git a/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/run-hex7gk4r.wandb b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/run-hex7gk4r.wandb new file mode 100644 index 0000000000000000000000000000000000000000..935ff1680c376e567309c7637b56a7c596387133 Binary files /dev/null and b/hex7gk4r/wandb/run-20250101_110303-hex7gk4r/run-hex7gk4r.wandb differ diff --git a/ibq29vgd/.hydra/config.yaml b/ibq29vgd/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05485f98b3c08bece875a460fe25a0cce7504f90 --- /dev/null +++ b/ibq29vgd/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1006_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/ibq29vgd/.hydra/hydra.yaml b/ibq29vgd/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a9c3ea33902a506df6688ba5bad8fbd236234cc --- /dev/null +++ b/ibq29vgd/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/ibq29vgd + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/ibq29vgd/.hydra/overrides.yaml b/ibq29vgd/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/ibq29vgd/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/ibq29vgd/eval_policy.log b/ibq29vgd/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..a36e581ea4563a9fdbdd1bc2ced984faf4e2e2b4 --- /dev/null +++ b/ibq29vgd/eval_policy.log @@ -0,0 +1,15 @@ +[2024-12-31 21:27:24,296][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-31 21:27:24,302][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-31 21:27:28,540][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/iz44h5u3/episode_rosbags/episode_1_2025-01-01-11-53-32.bag b/iz44h5u3/episode_rosbags/episode_1_2025-01-01-11-53-32.bag new file mode 100644 index 0000000000000000000000000000000000000000..31e5a975ff69d0674f8a8abc73acc5fcd4d4d9bc --- /dev/null +++ b/iz44h5u3/episode_rosbags/episode_1_2025-01-01-11-53-32.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c869bc8761f92b64c299364db9255afa1eaffae2a3b1eb0d594ab3b543e9ec58 +size 2572760162 diff --git a/iz44h5u3/episode_rosbags/episode_2_2025-01-01-11-54-28.bag b/iz44h5u3/episode_rosbags/episode_2_2025-01-01-11-54-28.bag new file mode 100644 index 0000000000000000000000000000000000000000..edd64f81e685f6decabae94b576dbc051b40c1ac --- /dev/null +++ b/iz44h5u3/episode_rosbags/episode_2_2025-01-01-11-54-28.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb6018dfd097285b0d18b1dc5b635ce7309f6c3ddbc5876c46edea6536ac8c42 +size 2607970632 diff --git a/iz44h5u3/episode_rosbags/episode_3_2025-01-01-11-55-18.bag b/iz44h5u3/episode_rosbags/episode_3_2025-01-01-11-55-18.bag new file mode 100644 index 0000000000000000000000000000000000000000..3844e160c3bb516b8b0794cddf6779733fa75bec --- /dev/null +++ b/iz44h5u3/episode_rosbags/episode_3_2025-01-01-11-55-18.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cbc269e86a5936e0f9c5681a07060d2048c7b32c7aaf195051826e1cb4058ee +size 2605505436 diff --git a/iz44h5u3/episode_rosbags/episode_4_2025-01-01-11-56-13.bag b/iz44h5u3/episode_rosbags/episode_4_2025-01-01-11-56-13.bag new file mode 100644 index 0000000000000000000000000000000000000000..8e5d709ff486b9fb1af21c7d1ec58e01c0c397d5 --- /dev/null +++ b/iz44h5u3/episode_rosbags/episode_4_2025-01-01-11-56-13.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86fce69e6a3b161ee5dd11be4b13ad2a257e22455fa506093bb2a16d59fdf8c0 +size 2596908407 diff --git a/k7tzwekb/episode_rosbags/cam_tf_world.npy b/k7tzwekb/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/k7tzwekb/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/k7tzwekb/episode_rosbags/depth_K.npy b/k7tzwekb/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/k7tzwekb/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/k7tzwekb/eval_robot.log b/k7tzwekb/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..2d7db18448925ce3324ca4dd459ccfc1c804d2c4 --- /dev/null +++ b/k7tzwekb/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-31 23:32:16,901][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-31 23:32:16,904][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/lydwde6h/episode_rosbags/episode_0_2025-01-01-11-16-24.bag b/lydwde6h/episode_rosbags/episode_0_2025-01-01-11-16-24.bag new file mode 100644 index 0000000000000000000000000000000000000000..98da0219668f3c4f71c69dbf03f3d80f9db2f797 --- /dev/null +++ b/lydwde6h/episode_rosbags/episode_0_2025-01-01-11-16-24.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fe3fb25e6cfc960b4ec283c87ea16452867e26ec6ed6b76b150cfaffabe16f9 +size 2603119767 diff --git a/lydwde6h/episode_rosbags/episode_1_2025-01-01-11-17-32.bag b/lydwde6h/episode_rosbags/episode_1_2025-01-01-11-17-32.bag new file mode 100644 index 0000000000000000000000000000000000000000..727b6919380017098b8985fa7526fbc5b12f8e77 --- /dev/null +++ b/lydwde6h/episode_rosbags/episode_1_2025-01-01-11-17-32.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3943dd411adfce5018f184b26232a9cbad85a8c2001544469466ac05307d255c +size 2136958362 diff --git a/wi30bsoe/.hydra/config.yaml b/wi30bsoe/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05485f98b3c08bece875a460fe25a0cce7504f90 --- /dev/null +++ b/wi30bsoe/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1006_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/wi30bsoe/.hydra/hydra.yaml b/wi30bsoe/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6bf61dd45043c4406c1630df6d7fedde6c0eec9 --- /dev/null +++ b/wi30bsoe/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/wi30bsoe + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/wi30bsoe/.hydra/overrides.yaml b/wi30bsoe/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/wi30bsoe/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/wi30bsoe/eval_policy.log b/wi30bsoe/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..0dcc3fa3c16df6ab705fbc935bc7cb6d1579a4b7 --- /dev/null +++ b/wi30bsoe/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-01 11:52:00,694][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 11:52:00,697][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-01 11:52:01,833][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) +