diff --git a/.gitattributes b/.gitattributes index 4e328a41ce9b3ce49fb0d9d00e314b629f74487a..d0a6237b748288883df2ba83581a6810db2da51e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -140,3 +140,51 @@ nl3lmlc3/episode_rosbags/episode_0_2025-01-04-21-34-30.bag filter=lfs diff=lfs m nl3lmlc3/episode_rosbags/episode_3_2025-01-04-21-37-04.bag filter=lfs diff=lfs merge=lfs -text 48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 filter=lfs diff=lfs merge=lfs -text p96n5ha2/episode_rosbags/episode_3_2025-01-03-23-01-30.bag filter=lfs diff=lfs merge=lfs -text +48t61yqi/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +p96n5ha2/episode_rosbags/episode_2_2025-01-03-23-00-42.bag filter=lfs diff=lfs merge=lfs -text +p96n5ha2/episode_rosbags/episode_0_2025-01-03-22-58-51.bag filter=lfs diff=lfs merge=lfs -text +p96n5ha2/episode_rosbags/episode_4_2025-01-03-23-02-30.bag filter=lfs diff=lfs merge=lfs -text +vgz4bwx7/episode_rosbags/episode_1_2025-01-03-21-21-55.bag filter=lfs diff=lfs merge=lfs -text +vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/2_eval_2_dab3f95c8b865ba8aed6.mp4 filter=lfs diff=lfs merge=lfs -text +vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/0_eval_0_27c97977eb157644346f.mp4 filter=lfs diff=lfs merge=lfs -text +vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/1_eval_1_ac781709866b92b2b17b.mp4 filter=lfs diff=lfs merge=lfs -text +vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/3_eval_3_c216389b31307ee9d9e4.mp4 filter=lfs diff=lfs merge=lfs -text +vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/4_eval_4_45ef389d8f994db06163.mp4 filter=lfs diff=lfs merge=lfs -text +vgz4bwx7/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +vgz4bwx7/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +vgz4bwx7/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +vgz4bwx7/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +vgz4bwx7/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +vgz4bwx7/episode_rosbags/episode_4_2025-01-03-21-24-10.bag filter=lfs diff=lfs merge=lfs -text +vgz4bwx7/episode_rosbags/episode_3_2025-01-03-21-23-25.bag filter=lfs diff=lfs merge=lfs -text +vgz4bwx7/episode_rosbags/episode_2_2025-01-03-21-22-35.bag filter=lfs diff=lfs merge=lfs -text +thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/1_eval_1_852c010e6bb3816412aa.mp4 filter=lfs diff=lfs merge=lfs -text +thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/2_eval_2_10e0e64153d3c9c65e4d.mp4 filter=lfs diff=lfs merge=lfs -text +thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/4_eval_4_50b70b008ecfe74e7155.mp4 filter=lfs diff=lfs merge=lfs -text +thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/3_eval_3_0730c18e4ed0ae46c605.mp4 filter=lfs diff=lfs merge=lfs -text +thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/0_eval_0_04d1dff61e6bb2dac431.mp4 filter=lfs diff=lfs merge=lfs -text +thh8eztz/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +thh8eztz/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +thh8eztz/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +thh8eztz/episode_rosbags/episode_4_2025-01-04-20-41-40.bag filter=lfs diff=lfs merge=lfs -text +thh8eztz/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +thh8eztz/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +pzxh5op7/episode_rosbags/episode_3_2025-01-04-22-23-57.bag filter=lfs diff=lfs merge=lfs -text +thh8eztz/episode_rosbags/episode_3_2025-01-04-20-41-03.bag filter=lfs diff=lfs merge=lfs -text +thh8eztz/episode_rosbags/episode_2_2025-01-04-20-40-16.bag filter=lfs diff=lfs merge=lfs -text +thh8eztz/episode_rosbags/episode_1_2025-01-04-20-39-11.bag filter=lfs diff=lfs merge=lfs -text +pzxh5op7/episode_rosbags/episode_1_2025-01-04-22-22-11.bag filter=lfs diff=lfs merge=lfs -text +ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/4_eval_4_ba85108b8a446815d41f.mp4 filter=lfs diff=lfs merge=lfs -text +ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/2_eval_2_ebd3ab9d4a47a5995972.mp4 filter=lfs diff=lfs merge=lfs -text +ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/0_eval_0_3881364072b3e77caced.mp4 filter=lfs diff=lfs merge=lfs -text +ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/1_eval_1_2f514b24221df96e8edb.mp4 filter=lfs diff=lfs merge=lfs -text +ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/3_eval_3_caa22dfc15ca57c8e31d.mp4 filter=lfs diff=lfs merge=lfs -text +pzxh5op7/episode_rosbags/episode_2_2025-01-04-22-22-58.bag filter=lfs diff=lfs merge=lfs -text +ts8zycr2/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ts8zycr2/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ts8zycr2/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ts8zycr2/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ts8zycr2/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +pzxh5op7/episode_rosbags/episode_4_2025-01-04-22-24-47.bag filter=lfs diff=lfs merge=lfs -text +pzxh5op7/episode_rosbags/episode_0_2025-01-04-22-21-23.bag filter=lfs diff=lfs merge=lfs -text +ts8zycr2/episode_rosbags/episode_4_2025-01-03-20-43-26.bag filter=lfs diff=lfs merge=lfs -text diff --git a/48t61yqi/eval_video/0_eval.mp4 b/48t61yqi/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..18919a3d9962cc192bf35e9e666f393804fce52d --- /dev/null +++ b/48t61yqi/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff4d172543858f403e01d883e2e919457e9b67113e1f51ac9eeb1c12fbd6c7b4 +size 1487478 diff --git a/dqnz7k8g/.hydra/config.yaml b/dqnz7k8g/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/dqnz7k8g/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/dqnz7k8g/.hydra/hydra.yaml b/dqnz7k8g/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be2aa40d7fb24a427d3153581a5dcf8e9720de76 --- /dev/null +++ b/dqnz7k8g/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/dqnz7k8g + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/dqnz7k8g/.hydra/overrides.yaml b/dqnz7k8g/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/dqnz7k8g/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/p96n5ha2/episode_rosbags/episode_0_2025-01-03-22-58-51.bag b/p96n5ha2/episode_rosbags/episode_0_2025-01-03-22-58-51.bag new file mode 100644 index 0000000000000000000000000000000000000000..c7a32b66353a2bf03f3b47f52d4f3ac682840e57 --- /dev/null +++ b/p96n5ha2/episode_rosbags/episode_0_2025-01-03-22-58-51.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3dbc1d6b24a60e47035a21f0a01c3617de681801ccd4d792ff20c24d92ebfcb7 +size 2609361105 diff --git a/p96n5ha2/episode_rosbags/episode_2_2025-01-03-23-00-42.bag b/p96n5ha2/episode_rosbags/episode_2_2025-01-03-23-00-42.bag new file mode 100644 index 0000000000000000000000000000000000000000..073129ae413690bd9904622564003decfad92393 --- /dev/null +++ b/p96n5ha2/episode_rosbags/episode_2_2025-01-03-23-00-42.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22fb57e450c43367cce4b70ea5ddc9dc870dd9537a9e6de09ea4aec16168578e +size 2610257384 diff --git a/p96n5ha2/episode_rosbags/episode_4_2025-01-03-23-02-30.bag b/p96n5ha2/episode_rosbags/episode_4_2025-01-03-23-02-30.bag new file mode 100644 index 0000000000000000000000000000000000000000..363cb26147aa621ab8cc163a3440cc812e79763d --- /dev/null +++ b/p96n5ha2/episode_rosbags/episode_4_2025-01-03-23-02-30.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9980f41ea14e3298850e5960bd1bed645af887334f6f26ebc604c8b1c61cb834 +size 2608981807 diff --git a/pzxh5op7/episode_rosbags/episode_0_2025-01-04-22-21-23.bag b/pzxh5op7/episode_rosbags/episode_0_2025-01-04-22-21-23.bag new file mode 100644 index 0000000000000000000000000000000000000000..0445cfa4e272e75ccb2673a0bebe4280edb7c3a6 --- /dev/null +++ b/pzxh5op7/episode_rosbags/episode_0_2025-01-04-22-21-23.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9662bf6d8f4c8fa63536dc7498b713453561232d022e20ca585dac9284fe2f85 +size 2629507544 diff --git a/pzxh5op7/episode_rosbags/episode_1_2025-01-04-22-22-11.bag b/pzxh5op7/episode_rosbags/episode_1_2025-01-04-22-22-11.bag new file mode 100644 index 0000000000000000000000000000000000000000..f35e749f81247aaf985e856bbef7c4fe5f1f497c --- /dev/null +++ b/pzxh5op7/episode_rosbags/episode_1_2025-01-04-22-22-11.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea935867df2e1e64d159b1ded56b378b96e9def60af740fe4373bcb148a875af +size 2601157168 diff --git a/pzxh5op7/episode_rosbags/episode_2_2025-01-04-22-22-58.bag b/pzxh5op7/episode_rosbags/episode_2_2025-01-04-22-22-58.bag new file mode 100644 index 0000000000000000000000000000000000000000..ecfdcbd6a6bab6d070e9bffb1f04055d2a80babb --- /dev/null +++ b/pzxh5op7/episode_rosbags/episode_2_2025-01-04-22-22-58.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f810c1b8d1ababe1331425d5c7dfde6128070280232eca5c3499c72a2386d947 +size 2614082614 diff --git a/pzxh5op7/episode_rosbags/episode_3_2025-01-04-22-23-57.bag b/pzxh5op7/episode_rosbags/episode_3_2025-01-04-22-23-57.bag new file mode 100644 index 0000000000000000000000000000000000000000..d90dcec583fc21b649dfd0b5ba75364871a998bc --- /dev/null +++ b/pzxh5op7/episode_rosbags/episode_3_2025-01-04-22-23-57.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a907e5005e3534338fdfc1dc530b88e4f0bc1e1f3ed36d9d1c79e54010aae6c +size 2613474178 diff --git a/pzxh5op7/episode_rosbags/episode_4_2025-01-04-22-24-47.bag b/pzxh5op7/episode_rosbags/episode_4_2025-01-04-22-24-47.bag new file mode 100644 index 0000000000000000000000000000000000000000..78bfd3a687d7ea5b1371b1dad7b84229770e4c7c --- /dev/null +++ b/pzxh5op7/episode_rosbags/episode_4_2025-01-04-22-24-47.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:090370aee96288c8db07631b7cdaac5da96235d4c5a966089a18822b0a061749 +size 2604392278 diff --git a/t2nzpp4t/episode_rosbags/cam_tf_world.npy b/t2nzpp4t/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/t2nzpp4t/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/t2nzpp4t/episode_rosbags/depth_K.npy b/t2nzpp4t/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/t2nzpp4t/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/t2nzpp4t/eval_robot.log b/t2nzpp4t/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..c602a27a0ecbc02578a8b7bf36913f731aeec517 --- /dev/null +++ b/t2nzpp4t/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 20:25:46,035][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 20:25:46,038][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/thh8eztz/.hydra/config.yaml b/thh8eztz/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/thh8eztz/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/thh8eztz/.hydra/hydra.yaml b/thh8eztz/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a425c86ed2dfeb9a512a729ec427dfd053268466 --- /dev/null +++ b/thh8eztz/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/thh8eztz/.hydra/overrides.yaml b/thh8eztz/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/thh8eztz/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/thh8eztz/episode_rosbags/aligned_depth_to_color_K.npy b/thh8eztz/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/thh8eztz/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/thh8eztz/episode_rosbags/cam_tf_world.npy b/thh8eztz/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/thh8eztz/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/thh8eztz/episode_rosbags/color_K.npy b/thh8eztz/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/thh8eztz/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/thh8eztz/episode_rosbags/depth_K.npy b/thh8eztz/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/thh8eztz/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/thh8eztz/episode_rosbags/episode_1_2025-01-04-20-39-11.bag b/thh8eztz/episode_rosbags/episode_1_2025-01-04-20-39-11.bag new file mode 100644 index 0000000000000000000000000000000000000000..0b917108ec7937d4a7763d4cdc4f1e9b93d1a58f --- /dev/null +++ b/thh8eztz/episode_rosbags/episode_1_2025-01-04-20-39-11.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6fdfd339186b61e0754980716970450a6252e5fc1d1be46b3f357fa3f043804 +size 1942728923 diff --git a/thh8eztz/episode_rosbags/episode_2_2025-01-04-20-40-16.bag b/thh8eztz/episode_rosbags/episode_2_2025-01-04-20-40-16.bag new file mode 100644 index 0000000000000000000000000000000000000000..ac0d9a636baa10747172c34eb39836fbaf96d86f --- /dev/null +++ b/thh8eztz/episode_rosbags/episode_2_2025-01-04-20-40-16.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33b372862a4241c8d17f072d18360c30e9309e99d500e2d090dadc822b4415a9 +size 1366413022 diff --git a/thh8eztz/episode_rosbags/episode_3_2025-01-04-20-41-03.bag b/thh8eztz/episode_rosbags/episode_3_2025-01-04-20-41-03.bag new file mode 100644 index 0000000000000000000000000000000000000000..a343a50da86a732d12290f199b9aa3d45e9f6a58 --- /dev/null +++ b/thh8eztz/episode_rosbags/episode_3_2025-01-04-20-41-03.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12239c80041708c226a41fbd024ada3347a8c017de039ad49beede8b8d5ec3e0 +size 1540328595 diff --git a/thh8eztz/episode_rosbags/episode_4_2025-01-04-20-41-40.bag b/thh8eztz/episode_rosbags/episode_4_2025-01-04-20-41-40.bag new file mode 100644 index 0000000000000000000000000000000000000000..f9a5ac6c2a687d67318096f7e2c76591256269db --- /dev/null +++ b/thh8eztz/episode_rosbags/episode_4_2025-01-04-20-41-40.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0d811c0c7e51513f99478dd9479e2caf3e82ad32aac27da5610dcf542546fee +size 1090351244 diff --git a/thh8eztz/eval_robot.log b/thh8eztz/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..c2a00a00478feb0a2e492a886c9b548421345444 --- /dev/null +++ b/thh8eztz/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 20:37:26,040][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 20:37:26,043][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/thh8eztz/eval_video/0_eval.mp4 b/thh8eztz/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a6c1cb919e48e556d60ba2deecb7f89c845ba8f9 --- /dev/null +++ b/thh8eztz/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04d1dff61e6bb2dac4311d430a65a69e02a8940b482ea5c88b6cba7b65ef0f52 +size 1474240 diff --git a/thh8eztz/eval_video/1_eval.mp4 b/thh8eztz/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..84395094bb3079ac9834b4be0a2a29c3f4adda37 --- /dev/null +++ b/thh8eztz/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:852c010e6bb3816412aace929a51b1e974f624c3f7c27938b37b0630efc22d31 +size 1205809 diff --git a/thh8eztz/eval_video/2_eval.mp4 b/thh8eztz/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f4f8264e8b95c1fc0aa16151c661941a4b7922b6 --- /dev/null +++ b/thh8eztz/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10e0e64153d3c9c65e4dc094da63ecf5967c1e7e88ef64d24787117143ec3d49 +size 920687 diff --git a/thh8eztz/eval_video/3_eval.mp4 b/thh8eztz/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6c2f7c2e19840187a364e79f5d2bb287a44a5913 --- /dev/null +++ b/thh8eztz/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0730c18e4ed0ae46c605b9dafb1aa70915e79195306fcecd873b8e3e7630e259 +size 1010993 diff --git a/thh8eztz/eval_video/4_eval.mp4 b/thh8eztz/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ea5b8f8a97b8dc3620c58072e0301e06f1bcd49c --- /dev/null +++ b/thh8eztz/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50b70b008ecfe74e7155654815b098ccfce640b366ae6bbe2caaf28d626e1004 +size 738236 diff --git a/thh8eztz/tb/events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 b/thh8eztz/tb/events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 new file mode 100644 index 0000000000000000000000000000000000000000..2c089d39882f70877aebe11a11d11027a3b3c85f --- /dev/null +++ b/thh8eztz/tb/events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d03e9c1108e6ed5fc4d32e5b28ce169925c698b2db8d86ffa2a7dd65e0805cd9 +size 6539 diff --git a/thh8eztz/wandb/debug-internal.log b/thh8eztz/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..68bb7e94ad223b1faf27a4f75585c57cdfb15332 --- /dev/null +++ b/thh8eztz/wandb/debug-internal.log @@ -0,0 +1,812 @@ +2025-01-04 20:37:32,036 INFO StreamThr :2013575 [internal.py:wandb_internal():85] W&B internal server running at pid: 2013575, started at: 2025-01-04 20:37:32.036237 +2025-01-04 20:37:32,038 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status +2025-01-04 20:37:32,051 INFO WriterThread:2013575 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/run-thh8eztz.wandb +2025-01-04 20:37:32,051 DEBUG SenderThread:2013575 [sender.py:send():379] send: header +2025-01-04 20:37:32,059 DEBUG SenderThread:2013575 [sender.py:send():379] send: run +2025-01-04 20:37:32,280 INFO SenderThread:2013575 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files +2025-01-04 20:37:32,280 INFO SenderThread:2013575 [sender.py:_start_run_threads():1188] run started: thh8eztz with start time 1736041052.036301 +2025-01-04 20:37:32,301 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 20:37:32,301 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: check_version +2025-01-04 20:37:32,350 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 20:37:32,355 DEBUG HandlerThread:2013575 [system_info.py:__init__():26] System info init +2025-01-04 20:37:32,355 DEBUG HandlerThread:2013575 [system_info.py:__init__():41] System info init done +2025-01-04 20:37:32,355 INFO HandlerThread:2013575 [system_monitor.py:start():194] Starting system monitor +2025-01-04 20:37:32,355 INFO SystemMonitor:2013575 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 20:37:32,355 INFO HandlerThread:2013575 [system_monitor.py:probe():214] Collecting system info +2025-01-04 20:37:32,356 INFO SystemMonitor:2013575 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 20:37:32,356 INFO SystemMonitor:2013575 [interfaces.py:start():188] Started disk monitoring +2025-01-04 20:37:32,357 INFO SystemMonitor:2013575 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 20:37:32,357 INFO SystemMonitor:2013575 [interfaces.py:start():188] Started memory monitoring +2025-01-04 20:37:32,358 INFO SystemMonitor:2013575 [interfaces.py:start():188] Started network monitoring +2025-01-04 20:37:32,368 DEBUG HandlerThread:2013575 [system_info.py:probe():152] Probing system +2025-01-04 20:37:32,370 DEBUG HandlerThread:2013575 [system_info.py:_probe_git():137] Probing git +2025-01-04 20:37:32,375 DEBUG HandlerThread:2013575 [system_info.py:_probe_git():145] Probing git done +2025-01-04 20:37:32,375 DEBUG HandlerThread:2013575 [system_info.py:probe():200] Probing system done +2025-01-04 20:37:32,375 DEBUG HandlerThread:2013575 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T01:37:32.369017', 'startedAt': '2025-01-05T01:37:32.028207', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4124.606125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.984, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.078, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 606.3384628295898}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 20:37:32,375 INFO HandlerThread:2013575 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 20:37:32,375 INFO HandlerThread:2013575 [system_monitor.py:probe():227] Publishing system info +2025-01-04 20:37:32,375 DEBUG HandlerThread:2013575 [system_info.py:_save_code():44] Saving code +2025-01-04 20:37:32,377 DEBUG HandlerThread:2013575 [system_info.py:_save_code():67] Saving code done +2025-01-04 20:37:32,377 DEBUG HandlerThread:2013575 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 20:37:32,438 DEBUG HandlerThread:2013575 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 20:37:32,438 INFO HandlerThread:2013575 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 20:37:32,442 DEBUG SenderThread:2013575 [sender.py:send():379] send: files +2025-01-04 20:37:32,442 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 20:37:32,442 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 20:37:32,442 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 20:37:32,680 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 20:37:32,681 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: python_packages +2025-01-04 20:37:32,682 DEBUG SenderThread:2013575 [sender.py:send():379] send: telemetry +2025-01-04 20:37:32,701 INFO wandb-upload_0:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/trl9szzv-wandb-metadata.json +2025-01-04 20:37:32,702 INFO wandb-upload_1:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/qktoycrx-code/FISH/eval_robot.py +2025-01-04 20:37:32,725 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:37:32,725 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:32,725 INFO HandlerThread:2013575 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/tb" + save: true +} +control { + relay_id: "46f399648cfc4418b20aca9cdc6e71fe" +} +_info { + stream_id: "thh8eztz" +} + +2025-01-04 20:37:32,725 DEBUG HandlerThread:2013575 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 20:37:32,726 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:37:32,735 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/vjeler6c-diff.patch +2025-01-04 20:37:32,839 DEBUG SenderThread:2013575 [sender.py:send():379] send: tbrecord +2025-01-04 20:37:32,840 DEBUG SenderThread:2013575 [sender.py:send():379] send: files +2025-01-04 20:37:32,840 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 with policy live +2025-01-04 20:37:33,027 INFO wandb-upload_0:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/njydtob1-events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:37:33,281 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/diff.patch +2025-01-04 20:37:33,281 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:37:33,282 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/requirements.txt +2025-01-04 20:37:33,282 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:37:33,282 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/code/FISH/eval_robot.py +2025-01-04 20:37:33,282 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-metadata.json +2025-01-04 20:37:33,282 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/code/FISH +2025-01-04 20:37:33,282 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/code +2025-01-04 20:37:33,681 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:34,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:35,280 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:37:35,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:36,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:37,681 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:37:37,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:38,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:39,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:40,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:41,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:42,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:37:42,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:43,683 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:44,683 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:45,683 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:46,683 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:47,681 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:37:47,681 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:37:47,721 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:47,799 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:37:48,683 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:49,466 DEBUG SenderThread:2013575 [sender.py:send():379] send: config +2025-01-04 20:37:49,683 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:50,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:51,285 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:37:51,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:52,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:53,189 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:37:53,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:54,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:55,286 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:37:55,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:56,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:57,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:58,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:37:58,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:59,287 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:37:59,685 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:00,685 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:01,288 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:38:01,685 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:02,681 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:38:02,681 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:38:02,725 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:03,289 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:38:03,686 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:03,739 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:04,289 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/config.yaml +2025-01-04 20:38:04,686 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:05,289 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:38:05,686 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:06,686 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:07,687 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:08,687 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:09,683 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:09,687 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:10,687 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:11,687 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:12,687 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:13,688 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:14,683 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:14,688 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:15,688 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:16,688 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:17,681 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:38:17,681 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:38:17,725 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:18,688 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:19,688 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:19,753 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:20,689 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:21,689 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:22,689 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:23,689 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:24,689 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:25,683 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:25,689 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:26,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:27,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:28,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:29,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:30,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:30,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:31,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:32,358 DEBUG SystemMonitor:2013575 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 20:38:32,359 DEBUG SenderThread:2013575 [sender.py:send():379] send: stats +2025-01-04 20:38:32,681 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:38:32,681 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:38:32,725 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:33,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:34,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:35,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:35,830 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:36,691 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:37,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:38,693 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:39,693 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:40,455 DEBUG SenderThread:2013575 [sender.py:send():379] send: files +2025-01-04 20:38:40,455 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_04d1dff61e6bb2dac431.mp4 with policy now +2025-01-04 20:38:40,455 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:38:40,456 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:38:40,456 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:38:40,457 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:38:40,694 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:40,741 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/ujd7fqor-media/videos/eval/0_eval_0_04d1dff61e6bb2dac431.mp4 +2025-01-04 20:38:41,298 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:38:41,298 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/0_eval_0_04d1dff61e6bb2dac431.mp4 +2025-01-04 20:38:41,299 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:38:41,299 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos +2025-01-04 20:38:41,299 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval +2025-01-04 20:38:41,299 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media +2025-01-04 20:38:41,472 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:41,591 INFO wandb-upload_0:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/oi317r9a-media/videos/eval/0_eval_0_04d1dff61e6bb2dac431.mp4 +2025-01-04 20:38:41,694 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:42,694 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:43,299 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:38:43,694 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:44,694 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:45,300 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:38:45,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:46,585 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:46,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:47,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:38:47,682 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:38:47,725 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:48,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:49,301 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:38:49,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:50,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:51,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:51,836 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:52,696 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:53,696 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:54,696 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:55,696 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:56,696 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:57,685 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:57,697 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:58,697 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:59,701 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:00,697 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:01,697 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:02,359 DEBUG SenderThread:2013575 [sender.py:send():379] send: stats +2025-01-04 20:39:02,681 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:39:02,682 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:39:02,729 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:02,797 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:03,697 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:04,697 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:05,698 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:06,698 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:07,737 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:08,307 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:39:08,547 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/e2wwpkkb-events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:39:08,698 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:08,713 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:09,307 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:39:09,698 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:10,704 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:11,308 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:39:11,700 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:12,701 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:13,309 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:39:13,700 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:13,807 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:14,700 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:15,310 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:39:15,700 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:16,700 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:17,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:39:17,682 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:39:17,725 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:18,701 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:19,701 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:19,739 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:20,701 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:21,701 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:22,701 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:23,701 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:24,702 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:25,686 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:25,702 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:26,702 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:27,702 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:28,702 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:29,702 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:30,702 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:31,686 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:31,703 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:32,360 DEBUG SenderThread:2013575 [sender.py:send():379] send: stats +2025-01-04 20:39:32,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:39:32,682 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:39:32,725 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:33,703 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:34,703 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:35,703 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:36,703 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:36,808 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:37,703 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:38,703 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:39,648 DEBUG SenderThread:2013575 [sender.py:send():379] send: files +2025-01-04 20:39:39,648 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_852c010e6bb3816412aa.mp4 with policy now +2025-01-04 20:39:39,654 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:39:39,654 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:39:39,654 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:39:39,655 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:39:39,703 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:39,908 INFO wandb-upload_0:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/mzyo1grh-media/videos/eval/1_eval_1_852c010e6bb3816412aa.mp4 +2025-01-04 20:39:40,317 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:39:40,317 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/1_eval_1_852c010e6bb3816412aa.mp4 +2025-01-04 20:39:40,318 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval +2025-01-04 20:39:40,586 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/0c4fxa9w-media/videos/eval/1_eval_1_852c010e6bb3816412aa.mp4 +2025-01-04 20:39:40,704 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:41,318 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:39:41,708 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:42,677 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:42,708 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:43,709 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:44,708 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:45,319 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:39:45,711 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:46,709 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:47,320 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:39:47,685 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:39:47,685 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:39:47,725 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:47,748 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:48,708 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:49,711 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:50,713 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:51,710 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:52,712 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:53,691 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:53,718 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:54,713 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:55,711 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:56,717 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:57,717 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:58,718 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:59,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:59,724 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:00,723 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:01,720 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:02,361 DEBUG SenderThread:2013575 [sender.py:send():379] send: stats +2025-01-04 20:40:02,686 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:40:02,686 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:40:02,729 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:03,724 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:04,720 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:04,738 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:05,723 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:06,724 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:07,722 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:08,723 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:09,722 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:10,691 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:10,727 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:11,728 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:12,726 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:13,196 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:40:13,196 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:40:13,196 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:40:13,328 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:40:13,328 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:40:13,563 INFO wandb-upload_0:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/p4b8hcom-events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:40:13,727 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:14,731 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:15,329 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:40:15,735 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:16,377 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:16,727 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:17,329 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:40:17,694 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:40:17,694 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:40:17,730 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:18,729 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:19,330 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:40:19,730 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:20,731 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:21,731 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:21,792 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:22,731 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:23,735 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:24,733 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:25,733 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:26,735 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:27,333 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:40:27,489 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:27,734 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:28,736 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:29,734 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:30,735 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:31,735 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:32,361 DEBUG SenderThread:2013575 [sender.py:send():379] send: stats +2025-01-04 20:40:32,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:40:32,695 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:40:32,741 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:32,766 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:33,335 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:40:33,737 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:34,736 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:35,735 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:36,739 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:37,164 DEBUG SenderThread:2013575 [sender.py:send():379] send: files +2025-01-04 20:40:37,164 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_10e0e64153d3c9c65e4d.mp4 with policy now +2025-01-04 20:40:37,165 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:40:37,166 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:40:37,166 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:40:37,166 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:40:37,336 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:40:37,337 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:40:37,337 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/2_eval_2_10e0e64153d3c9c65e4d.mp4 +2025-01-04 20:40:37,337 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval +2025-01-04 20:40:37,424 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/tutyof3y-media/videos/eval/2_eval_2_10e0e64153d3c9c65e4d.mp4 +2025-01-04 20:40:37,687 INFO wandb-upload_0:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/qb6w7ex6-media/videos/eval/2_eval_2_10e0e64153d3c9c65e4d.mp4 +2025-01-04 20:40:37,740 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:38,699 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:38,743 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:39,337 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:40:39,738 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:40,765 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:41,741 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:42,741 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:43,338 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:40:43,738 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:44,549 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:44,742 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:45,339 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:40:45,745 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:46,748 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:47,694 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:40:47,695 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:40:47,752 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:48,749 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:49,745 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:49,757 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:50,749 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:51,756 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:52,748 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:53,755 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:54,751 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:55,696 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:55,758 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:56,757 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:57,757 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:58,759 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:59,756 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:00,025 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:41:00,026 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:41:00,026 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:41:00,344 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:41:00,344 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:41:00,766 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:01,344 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:01,593 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:01,769 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:02,362 DEBUG SenderThread:2013575 [sender.py:send():379] send: stats +2025-01-04 20:41:02,696 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:41:02,697 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:41:02,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:03,345 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:03,768 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:04,769 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:05,346 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:05,765 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:06,765 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:07,224 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:07,346 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:07,765 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:08,766 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:09,766 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:10,766 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:11,785 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:12,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:12,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:13,348 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:13,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:14,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:15,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:16,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:17,694 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:41:17,694 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:41:17,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:17,826 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:18,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:19,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:20,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:21,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:22,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:23,696 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:23,768 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:24,768 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:25,809 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:26,785 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:26,869 DEBUG SenderThread:2013575 [sender.py:send():379] send: files +2025-01-04 20:41:26,869 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_0730c18e4ed0ae46c605.mp4 with policy now +2025-01-04 20:41:26,872 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:41:26,872 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:41:26,872 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:41:26,873 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:41:27,131 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/1ffl4gt0-media/videos/eval/3_eval_3_0730c18e4ed0ae46c605.mp4 +2025-01-04 20:41:27,353 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:27,353 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:41:27,353 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/3_eval_3_0730c18e4ed0ae46c605.mp4 +2025-01-04 20:41:27,354 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval +2025-01-04 20:41:27,647 INFO wandb-upload_0:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/cyjt0a22-media/videos/eval/3_eval_3_0730c18e4ed0ae46c605.mp4 +2025-01-04 20:41:27,771 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:28,697 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:28,775 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:29,354 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:29,784 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:30,785 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:31,355 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:31,784 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:32,362 DEBUG SenderThread:2013575 [sender.py:send():379] send: stats +2025-01-04 20:41:32,700 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:41:32,701 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:41:32,788 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:33,790 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:34,177 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:34,785 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:35,356 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:35,790 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:36,787 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:37,070 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:41:37,070 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:41:37,070 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:41:37,357 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:41:37,357 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:41:37,593 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/8ds8qg2u-events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:41:37,787 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:38,795 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:39,357 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:39,796 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:39,853 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:40,790 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:41,358 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:41,788 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:42,788 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:43,359 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:43,788 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:44,788 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:45,696 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:45,788 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:46,788 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:47,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:41:47,695 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:41:47,788 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:48,789 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:49,789 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:50,789 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:50,834 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:51,789 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:52,789 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:53,362 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:53,789 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:54,789 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:55,790 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:56,615 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:56,790 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:57,364 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:57,644 DEBUG SenderThread:2013575 [sender.py:send():379] send: files +2025-01-04 20:41:57,644 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_50b70b008ecfe74e7155.mp4 with policy now +2025-01-04 20:41:57,650 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:41:57,650 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:41:57,650 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:41:57,650 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:41:57,790 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:57,889 INFO wandb-upload_0:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/6ir135qj-media/videos/eval/4_eval_4_50b70b008ecfe74e7155.mp4 +2025-01-04 20:41:58,364 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:41:58,364 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/4_eval_4_50b70b008ecfe74e7155.mp4 +2025-01-04 20:41:58,364 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval +2025-01-04 20:41:58,625 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/xbl5egrt-media/videos/eval/4_eval_4_50b70b008ecfe74e7155.mp4 +2025-01-04 20:41:58,799 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:59,364 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:59,804 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:00,805 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:01,809 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:01,933 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:42:02,363 DEBUG SenderThread:2013575 [sender.py:send():379] send: stats +2025-01-04 20:42:02,707 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:42:02,707 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:42:02,812 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:03,366 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:42:03,808 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:04,816 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:05,366 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:42:05,811 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:06,824 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:07,706 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:42:07,810 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:08,817 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:09,814 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:10,814 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:11,820 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:12,707 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:42:12,818 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:13,820 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:14,820 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:15,825 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:16,822 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:17,708 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:42:17,715 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:42:17,715 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:42:17,828 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:18,831 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:19,828 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:20,126 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:42:20,126 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:20,126 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:20,371 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:42:20,372 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:42:20,556 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 20:42:20,556 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 20:42:20,754 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:42:20,823 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:21,021 INFO wandb-upload_0:2013575 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp3y2jujh2 +2025-01-04 20:42:21,162 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 20:42:21,172 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:42:21,372 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:42:21,372 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:42:21,372 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/table/eval/total_success_rate_ci_6_f73c6106c1ced0c2a756.table.json +2025-01-04 20:42:21,372 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 20:42:21,372 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media +2025-01-04 20:42:21,372 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/table +2025-01-04 20:42:21,373 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/table/eval +2025-01-04 20:42:21,398 INFO SenderThread:2013575 [sender.py:send_request_log_artifact():1525] logged artifact run-thh8eztz-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTExMTQ0Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MDIyMg==', 'latestArtifact': None}} +2025-01-04 20:42:21,398 DEBUG SenderThread:2013575 [sender.py:send():379] send: files +2025-01-04 20:42:21,398 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json with policy now +2025-01-04 20:42:21,398 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:42:21,400 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,402 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,402 DEBUG SenderThread:2013575 [sender.py:send():379] send: telemetry +2025-01-04 20:42:21,402 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,402 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,402 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,403 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,403 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,403 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,403 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,404 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,404 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,404 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,405 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,405 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,405 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,405 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,405 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,406 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,406 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,406 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,406 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,407 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,407 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,407 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,408 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 20:42:21,583 INFO wandb-upload_1:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/5ri5di85-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 20:42:21,616 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/7f1q6zll-media/table/eval/total_success_rate_ci_6_f73c6106c1ced0c2a756.table.json +2025-01-04 20:42:21,758 INFO wandb-upload_0:2013575 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpomz4jqfw +2025-01-04 20:42:21,818 INFO wandb-upload_1:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/hwwdg8aj-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 20:42:21,823 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:22,143 INFO SenderThread:2013575 [sender.py:send_request_log_artifact():1525] logged artifact run-thh8eztz-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTExMTQ1OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MDIyNg==', 'latestArtifact': None}} +2025-01-04 20:42:22,143 DEBUG SenderThread:2013575 [sender.py:send():379] send: files +2025-01-04 20:42:22,143 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_f73c6106c1ced0c2a756.table.json with policy now +2025-01-04 20:42:22,143 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:42:22,143 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:22,144 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:22,372 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:42:22,375 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/u2xg3bu1-media/table/eval/total_success_rate_ci_6_f73c6106c1ced0c2a756.table.json +2025-01-04 20:42:22,824 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:23,373 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:42:23,701 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:42:23,824 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:24,824 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:25,824 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:26,824 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:27,824 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:28,824 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:29,705 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:42:29,825 DEBUG SenderThread:2013575 [sender.py:send():379] send: exit +2025-01-04 20:42:29,825 INFO SenderThread:2013575 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 20:42:29,825 INFO SenderThread:2013575 [sender.py:send_exit():588] handling runtime: 297 +2025-01-04 20:42:29,826 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:29,827 INFO SenderThread:2013575 [sender.py:send_exit():594] send defer +2025-01-04 20:42:29,827 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:29,827 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 20:42:29,827 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:29,827 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 20:42:29,827 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 1 +2025-01-04 20:42:29,827 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:29,827 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 20:42:29,827 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:29,827 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 20:42:29,827 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 2 +2025-01-04 20:42:29,827 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:29,827 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 20:42:29,827 INFO HandlerThread:2013575 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 20:42:29,828 INFO HandlerThread:2013575 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 20:42:29,828 DEBUG SystemMonitor:2013575 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 20:42:29,828 INFO HandlerThread:2013575 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 20:42:29,828 DEBUG SystemMonitor:2013575 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 20:42:29,885 INFO HandlerThread:2013575 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 20:42:29,885 INFO HandlerThread:2013575 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 20:42:29,886 INFO HandlerThread:2013575 [interfaces.py:finish():200] Joined network monitor +2025-01-04 20:42:29,886 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:29,886 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 20:42:29,886 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 3 +2025-01-04 20:42:29,886 DEBUG SenderThread:2013575 [sender.py:send():379] send: stats +2025-01-04 20:42:29,886 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:29,886 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 20:42:29,887 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:29,887 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 20:42:29,887 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 4 +2025-01-04 20:42:29,887 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:29,887 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 20:42:30,375 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/config.yaml +2025-01-04 20:42:30,376 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:42:31,376 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:42:36,126 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:42:36,126 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:42:36,126 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 20:42:36,127 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:36,127 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 20:42:36,127 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 5 +2025-01-04 20:42:36,127 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:42:36,127 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:42:36,127 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:36,128 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:36,128 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 20:42:36,128 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:36,129 DEBUG SenderThread:2013575 [sender.py:send():379] send: summary +2025-01-04 20:42:36,132 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:36,133 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:36,133 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 20:42:36,133 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 6 +2025-01-04 20:42:36,133 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:36,133 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 20:42:36,133 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:36,133 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 20:42:36,133 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 7 +2025-01-04 20:42:36,133 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:42:36,133 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:36,133 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 20:42:36,134 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:36,134 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 20:42:36,378 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:42:36,826 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:42:38,952 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 8 +2025-01-04 20:42:38,952 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:42:38,952 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:38,952 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 20:42:38,952 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:38,952 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 20:42:38,952 INFO SenderThread:2013575 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 20:42:38,955 INFO SenderThread:2013575 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 20:42:38,957 INFO SenderThread:2013575 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 20:42:38,969 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 9 +2025-01-04 20:42:38,969 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:38,969 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 20:42:38,969 DEBUG SenderThread:2013575 [sender.py:send():379] send: artifact +2025-01-04 20:42:39,303 INFO wandb-upload_2:2013575 [upload_job.py:push():85] Skipped uploading /tmp/tmp0m470vzd/wandb-job.json +2025-01-04 20:42:39,304 INFO wandb-upload_0:2013575 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpobqrfexr +2025-01-04 20:42:39,379 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:42:39,432 INFO wandb-upload_1:2013575 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_0vhxu1f +2025-01-04 20:42:39,827 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:42:39,956 INFO SenderThread:2013575 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTExMTc3MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEwMTI1NQ==', 'versionIndex': 416}}} +2025-01-04 20:42:39,956 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:39,956 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 20:42:39,956 INFO SenderThread:2013575 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 20:42:40,380 INFO SenderThread:2013575 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files +2025-01-04 20:42:40,380 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/config.yaml config.yaml +2025-01-04 20:42:40,380 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json wandb-summary.json +2025-01-04 20:42:40,380 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/requirements.txt requirements.txt +2025-01-04 20:42:40,380 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-metadata.json wandb-metadata.json +2025-01-04 20:42:40,381 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:42:40,381 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log output.log +2025-01-04 20:42:40,384 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/diff.patch diff.patch +2025-01-04 20:42:40,386 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 20:42:40,389 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/2_eval_2_10e0e64153d3c9c65e4d.mp4 media/videos/eval/2_eval_2_10e0e64153d3c9c65e4d.mp4 +2025-01-04 20:42:40,389 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/3_eval_3_0730c18e4ed0ae46c605.mp4 media/videos/eval/3_eval_3_0730c18e4ed0ae46c605.mp4 +2025-01-04 20:42:40,389 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/0_eval_0_04d1dff61e6bb2dac431.mp4 media/videos/eval/0_eval_0_04d1dff61e6bb2dac431.mp4 +2025-01-04 20:42:40,389 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/1_eval_1_852c010e6bb3816412aa.mp4 media/videos/eval/1_eval_1_852c010e6bb3816412aa.mp4 +2025-01-04 20:42:40,389 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/4_eval_4_50b70b008ecfe74e7155.mp4 media/videos/eval/4_eval_4_50b70b008ecfe74e7155.mp4 +2025-01-04 20:42:40,389 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 20:42:40,389 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/table/eval/total_success_rate_ci_6_f73c6106c1ced0c2a756.table.json media/table/eval/total_success_rate_ci_6_f73c6106c1ced0c2a756.table.json +2025-01-04 20:42:40,389 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 10 +2025-01-04 20:42:40,389 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:42:40,390 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:40,390 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 20:42:40,390 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:40,390 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 20:42:40,390 INFO SenderThread:2013575 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 20:42:40,570 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/config.yaml +2025-01-04 20:42:40,574 INFO wandb-upload_1:2013575 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/requirements.txt +2025-01-04 20:42:40,598 INFO wandb-upload_0:2013575 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:42:40,604 INFO wandb-upload_4:2013575 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:42:40,664 INFO wandb-upload_3:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/1j0f0tqh-events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:42:40,828 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:42:40,828 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:42:40,864 INFO Thread-11 (_thread_body):2013575 [sender.py:transition_state():614] send defer: 11 +2025-01-04 20:42:40,864 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:40,864 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 20:42:40,865 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:40,865 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 20:42:40,865 INFO SenderThread:2013575 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 20:42:40,866 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 12 +2025-01-04 20:42:40,866 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:40,866 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 20:42:40,866 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:40,866 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 20:42:40,866 INFO SenderThread:2013575 [file_stream.py:finish():601] file stream finish called +2025-01-04 20:42:41,136 INFO SenderThread:2013575 [file_stream.py:finish():605] file stream finish is done +2025-01-04 20:42:41,136 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 13 +2025-01-04 20:42:41,136 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:41,136 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 20:42:41,137 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:41,137 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 20:42:41,137 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 14 +2025-01-04 20:42:41,137 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:41,137 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 20:42:41,137 DEBUG SenderThread:2013575 [sender.py:send():379] send: final +2025-01-04 20:42:41,137 DEBUG SenderThread:2013575 [sender.py:send():379] send: footer +2025-01-04 20:42:41,137 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:41,137 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 20:42:41,138 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:42:41,138 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:42:41,138 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 20:42:41,139 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:42:41,139 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 20:42:41,139 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: server_info +2025-01-04 20:42:41,141 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 20:42:41,141 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:41,202 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:42:41,202 INFO MainThread:2013575 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 20:42:41,203 INFO MainThread:2013575 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 20:42:41,203 INFO MainThread:2013575 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 20:42:41,203 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 20:42:41,203 INFO HandlerThread:2013575 [handler.py:finish():882] shutting down handler +2025-01-04 20:42:42,139 INFO WriterThread:2013575 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/run-thh8eztz.wandb +2025-01-04 20:42:42,202 INFO SenderThread:2013575 [sender.py:finish():1615] shutting down sender +2025-01-04 20:42:42,203 INFO SenderThread:2013575 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 20:42:42,203 INFO SenderThread:2013575 [file_pusher.py:join():175] waiting for file pusher diff --git a/thh8eztz/wandb/debug.log b/thh8eztz/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..5b925fbf2e628de7cafcdb1ad244ff1c6d58a646 --- /dev/null +++ b/thh8eztz/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 20:37:32,030 INFO MainThread:2013380 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 20:37:32,030 INFO MainThread:2013380 [wandb_setup.py:_flush():76] Configure stats pid to 2013380 +2025-01-04 20:37:32,030 INFO MainThread:2013380 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 20:37:32,030 INFO MainThread:2013380 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/settings +2025-01-04 20:37:32,030 INFO MainThread:2013380 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 20:37:32,030 INFO MainThread:2013380 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 20:37:32,030 INFO MainThread:2013380 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 20:37:32,030 INFO MainThread:2013380 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 20:37:32,031 INFO MainThread:2013380 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/logs/debug.log +2025-01-04 20:37:32,031 INFO MainThread:2013380 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/logs/debug-internal.log +2025-01-04 20:37:32,031 INFO MainThread:2013380 [wandb_init.py:init():569] calling init triggers +2025-01-04 20:37:32,031 INFO MainThread:2013380 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz'} +2025-01-04 20:37:32,031 INFO MainThread:2013380 [wandb_init.py:init():619] starting backend +2025-01-04 20:37:32,031 INFO MainThread:2013380 [wandb_init.py:init():623] setting up manager +2025-01-04 20:37:32,035 INFO MainThread:2013380 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 20:37:32,036 INFO MainThread:2013380 [wandb_init.py:init():631] backend started and connected +2025-01-04 20:37:32,050 INFO MainThread:2013380 [wandb_init.py:init():720] updated telemetry +2025-01-04 20:37:32,057 INFO MainThread:2013380 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 20:37:32,300 INFO MainThread:2013380 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 20:37:32,339 INFO MainThread:2013380 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 20:37:32,339 INFO MainThread:2013380 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 20:37:32,681 INFO MainThread:2013380 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 20:37:32,681 INFO MainThread:2013380 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 20:37:32,681 INFO MainThread:2013380 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 20:37:32,681 INFO MainThread:2013380 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 20:37:32,683 INFO MainThread:2013380 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 20:37:32,684 INFO MainThread:2013380 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/tb, True +2025-01-04 20:37:49,466 INFO MainThread:2013380 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-04 20:42:42,204 WARNING MsgRouterThr:2013380 [router.py:message_loop():77] message_loop has been closed diff --git a/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/code/FISH/eval_robot.py b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/config.yaml b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e448bff411e51b5f3b1f18b9351849dd3e32032f --- /dev/null +++ b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1926_1' +wandb_notes: + desc: null + value: 1926_1_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736041052 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/diff.patch b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b5be3e37ff49c316e2685f8bd605106c30e9bfb6 --- /dev/null +++ b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..27e7d93 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json new file mode 100644 index 0000000000000000000000000000000000000000..faa9910a25d3f0826c10a7b81de69b8b94278cd1 --- /dev/null +++ b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5363743901179997, 0, 0.48415964842159465, 0, 0.21277225023292168, 0, 0.1673506349990479]]} \ No newline at end of file diff --git a/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/table/eval/total_success_rate_ci_6_f73c6106c1ced0c2a756.table.json b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/table/eval/total_success_rate_ci_6_f73c6106c1ced0c2a756.table.json new file mode 100644 index 0000000000000000000000000000000000000000..e6323427934d83d5f9e698f3c056069fa6f87e13 --- /dev/null +++ b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/table/eval/total_success_rate_ci_6_f73c6106c1ced0c2a756.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[16, 35, 0.45714285714285713, 0.2863466442058624, 0.6446045636841529, 0.29377636317562883, 0.613848760701298, 0.31371987614780983, 0.5979055166048695, 0.3634386435424223, 0.5717472731923613]]} \ No newline at end of file diff --git a/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/0_eval_0_04d1dff61e6bb2dac431.mp4 b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/0_eval_0_04d1dff61e6bb2dac431.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a6c1cb919e48e556d60ba2deecb7f89c845ba8f9 --- /dev/null +++ b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/0_eval_0_04d1dff61e6bb2dac431.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04d1dff61e6bb2dac4311d430a65a69e02a8940b482ea5c88b6cba7b65ef0f52 +size 1474240 diff --git a/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/1_eval_1_852c010e6bb3816412aa.mp4 b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/1_eval_1_852c010e6bb3816412aa.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..84395094bb3079ac9834b4be0a2a29c3f4adda37 --- /dev/null +++ b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/1_eval_1_852c010e6bb3816412aa.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:852c010e6bb3816412aace929a51b1e974f624c3f7c27938b37b0630efc22d31 +size 1205809 diff --git a/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/2_eval_2_10e0e64153d3c9c65e4d.mp4 b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/2_eval_2_10e0e64153d3c9c65e4d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f4f8264e8b95c1fc0aa16151c661941a4b7922b6 --- /dev/null +++ b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/2_eval_2_10e0e64153d3c9c65e4d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10e0e64153d3c9c65e4dc094da63ecf5967c1e7e88ef64d24787117143ec3d49 +size 920687 diff --git a/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/3_eval_3_0730c18e4ed0ae46c605.mp4 b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/3_eval_3_0730c18e4ed0ae46c605.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6c2f7c2e19840187a364e79f5d2bb287a44a5913 --- /dev/null +++ b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/3_eval_3_0730c18e4ed0ae46c605.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0730c18e4ed0ae46c605b9dafb1aa70915e79195306fcecd873b8e3e7630e259 +size 1010993 diff --git a/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/4_eval_4_50b70b008ecfe74e7155.mp4 b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/4_eval_4_50b70b008ecfe74e7155.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ea5b8f8a97b8dc3620c58072e0301e06f1bcd49c --- /dev/null +++ b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/4_eval_4_50b70b008ecfe74e7155.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50b70b008ecfe74e7155654815b098ccfce640b366ae6bbe2caaf28d626e1004 +size 738236 diff --git a/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..de714ff75d91ab7168db184a7fdecd2628fe4d41 --- /dev/null +++ b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log @@ -0,0 +1,269 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736041069.467021]: resetting environment +[INFO] [1736041069.473312]: cleared current plan +[INFO] [1736041069.474370]: moving to home +[INFO] [1736041070.476657]: reached home +[INFO] [1736041070.478333]: reset action history +[INFO] [1736041073.188757]: environment reset +Starting episode 0 +[INFO] [1736041073.194237]: resetting environment +[INFO] [1736041073.202464]: cleared current plan +[INFO] [1736041073.203618]: moving to home +[INFO] [1736041074.205281]: reached home +[INFO] [1736041074.206747]: reset action history +[INFO] [1736041076.915775]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1736041080.474305]: resetting environment +[INFO] [1736041080.479667]: cleared current plan +[INFO] [1736041080.480699]: moving to home +[INFO] [1736041081.482830]: reached home +[INFO] [1736041081.484060]: reset action history +[INFO] [1736041084.195374]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/episode_rosbags/episode_0_2025-01-04-20-38-04.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2013380.log +started roslaunch server http://158.130.50.37:39111/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2013685] +process[print_text-2]: started with pid [2013709] +process[rosbag_record-3]: started with pid [2013710] +started bagging! +[INFO] [1736041119.450974]: Storing episode... +[pub_text-1] killing on exit +[print_text-2] killing on exit[rosbag_record-3] killing on exit +[INFO] [1736041120.250760]: Stored episode 1. +[INFO] [1736041120.251015]: Saving video... +[INFO] [1736041120.442480]: Video saved! +Starting episode 1 +[INFO] [1736041120.458858]: resetting environment +[INFO] [1736041120.471062]: cleared current plan +[INFO] [1736041120.471861]: moving to home +[INFO] [1736041123.875047]: reached home +[INFO] [1736041123.875506]: reset action history +[INFO] [1736041126.585054]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736041147.703118]: resetting environment +[INFO] [1736041147.711676]: cleared current plan +[INFO] [1736041147.712209]: moving to home +[INFO] [1736041148.713348]: reached home +[INFO] [1736041148.713807]: reset action history +[INFO] [1736041151.424888]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/episode_rosbags/episode_1_2025-01-04-20-39-11.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2013380.log +started roslaunch server http://158.130.50.37:36695/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2013927] +process[print_text-5]: started with pid [2013928] +process[rosbag_record-6]: started with pid [2013929] +started bagging! +preempting episode +[INFO] [1736041178.662833]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736041179.409072]: Stored episode 2. +[INFO] [1736041179.411099]: Saving video... +[INFO] [1736041179.630375]: Video saved! +Starting episode 2 +[INFO] [1736041179.664500]: resetting environment +[INFO] [1736041179.676008]: cleared current plan +[INFO] [1736041179.676890]: moving to home +[INFO] [1736041182.979206]: reached home +[INFO] [1736041182.983009]: reset action history +[INFO] [1736041185.750603]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736041212.588037]: resetting environment +[INFO] [1736041212.595289]: cleared current plan +[INFO] [1736041212.595507]: moving to home +[INFO] [1736041213.596976]: reached home +[INFO] [1736041213.600075]: reset action history +[INFO] [1736041216.370767]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/episode_rosbags/episode_2_2025-01-04-20-40-16.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2013380.log +started roslaunch server http://158.130.50.37:41629/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2014394] +process[print_text-8]: started with pid [2014395] +process[rosbag_record-9]: started with pid [2014396] +started bagging! +For topic gripper_width: timestamp difference is -109827851 for nearest: 1736041225251576806 - target: 1736041225361404657 at idx 109 +For topic gripper_width: timestamp difference is 101916195 for nearest: 1736041231251581789 - target: 1736041231149665594 at idx 189 +For topic gripper_width: timestamp difference is 111012237 for nearest: 1736041235718247430 - target: 1736041235607235193 at idx 243 +preempting episode +[INFO] [1736041236.062134]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1736041236.836289]: Stored episode 3. +[INFO] [1736041236.839368]: Saving video... +[INFO] [1736041237.125535]: Video saved! +Starting episode 3 +[INFO] [1736041237.180511]: resetting environment +[INFO] [1736041237.214582]: cleared current plan +[INFO] [1736041237.218212]: moving to home +[INFO] [1736041240.721243]: reached home +[INFO] [1736041240.723420]: reset action history +[INFO] [1736041243.501253]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736041259.525781]: resetting environment +[INFO] [1736041259.547973]: cleared current plan +[INFO] [1736041259.548254]: moving to home +[INFO] [1736041260.551123]: reached home +[INFO] [1736041260.554795]: reset action history +[INFO] [1736041263.337402]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/episode_rosbags/episode_3_2025-01-04-20-41-03.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2013380.log +started roslaunch server http://158.130.50.37:38055/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2014600] +process[print_text-11]: started with pid [2014601] +process[rosbag_record-12]: started with pid [2014602] +started bagging! +For topic gripper_width: timestamp difference is 105407853 for nearest: 1736041271718245644 - target: 1736041271612837791 at idx 121 +preempting episode +[INFO] [1736041285.767829]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit[print_text-11] killing on exit +[INFO] [1736041286.586807]: Stored episode 4. +[INFO] [1736041286.592726]: Saving video... +[WARN] [1736041286.880613]: Plan exhausted +[INFO] [1736041286.825411]: Video saved! +Starting episode 4 +[INFO] [1736041286.877607]: resetting environment +[INFO] [1736041286.919644]: cleared current plan +[INFO] [1736041286.920249]: moving to home +[INFO] [1736041290.330147]: reached home +[INFO] [1736041290.335952]: reset action history +[INFO] [1736041293.130605]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736041296.764370]: resetting environment +[INFO] [1736041296.799928]: cleared current plan +[INFO] [1736041296.800864]: moving to home +[INFO] [1736041297.807339]: reached home +[INFO] [1736041297.814454]: reset action history +[INFO] [1736041300.591527]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/episode_rosbags/episode_4_2025-01-04-20-41-40.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2013380.log +started roslaunch server http://158.130.50.37:42599/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2014804] +process[print_text-14]: started with pid [2014805] +process[rosbag_record-15]: started with pid [2014806] +started bagging! +For topic gripper_width: timestamp difference is 100453392 for nearest: 1736041311118246094 - target: 1736041311017792702 at idx 154 +preempting episode +[INFO] [1736041316.624904]: Storing episode... +[pub_text-13] killing on exit +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[INFO] [1736041317.384374]: Stored episode 5. +[INFO] [1736041317.391420]: Saving video... +[INFO] [1736041317.604153]: Video saved! +[INFO] [1736041317.650714]: resetting environment +[INFO] [1736041317.682550]: cleared current plan +[INFO] [1736041317.682761]: moving to home +[INFO] [1736041320.889611]: reached home +[INFO] [1736041320.893353]: reset action history +[INFO] [1736041323.678982]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/requirements.txt b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-metadata.json b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c30e1ef8e351805b14265bc60d01272ff94b4a2f --- /dev/null +++ b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T01:37:32.369017", + "startedAt": "2025-01-05T01:37:32.028207", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4124.606125, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.984, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.078, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 606.3384628295898 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..bd8f3f1b21feca750d6f2135895b718303aa84e6 --- /dev/null +++ b/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "04d1dff61e6bb2dac4311d430a65a69e02a8940b482ea5c88b6cba7b65ef0f52", "size": 1474240, "path": "media/videos/eval/0_eval_0_04d1dff61e6bb2dac431.mp4"}, "global_step": 436, "_timestamp": 1736041341.0421555, "_runtime": 289.0058546066284, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "852c010e6bb3816412aace929a51b1e974f624c3f7c27938b37b0630efc22d31", "size": 1205809, "path": "media/videos/eval/1_eval_1_852c010e6bb3816412aa.mp4"}, "eval/num_success": 0.0, "episode": 35.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5363743901252747, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4841596484184265, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.21277225017547607, "eval/success_rate_lb_0.1": 0.0, "eval/success_rate_ub_0.1": 0.167350634932518, "eval/2_eval": {"_type": "video-file", "sha256": "10e0e64153d3c9c65e4dc094da63ecf5967c1e7e88ef64d24787117143ec3d49", "size": 920687, "path": "media/videos/eval/2_eval_2_10e0e64153d3c9c65e4d.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "0730c18e4ed0ae46c605b9dafb1aa70915e79195306fcecd873b8e3e7630e259", "size": 1010993, "path": "media/videos/eval/3_eval_3_0730c18e4ed0ae46c605.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "50b70b008ecfe74e7155654815b098ccfce640b366ae6bbe2caaf28d626e1004", "size": 738236, "path": "media/videos/eval/4_eval_4_50b70b008ecfe74e7155.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "e6508f0181aa84573747f0d380a2c6725005451ce9eb7fcdf77e1f39c90f7528", "size": 332, "artifact_path": "wandb-client-artifact://rjjzwuoustgd8h8ll0f090cltnv210rxp1tn770cug68a2jud5d35e8hwu2y6yod411jgwfjjjkqdd09tyoejekpwtplqbrp94sp14drtpl3y0n2n9s1f97bkcdm5x3d/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://17mhmagkaiqvv3zxh9nrl41w0otfgfj0qttfm2l0l7tbyar2wj955zs53gqmoiu7de7th30x00b0o7k7lt5atnopvn9eu515hvla7gj5vjr0w80uarwi8ihojxd5vw4w:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 16, "total_num_episodes": 35, "total_success_rate": 0.45714285714285713, "total_success_rate_lb_0.01": 0.2863466442058624, "total_success_rate_ub_0.01": 0.6446045636841529, "total_success_rate_lb_0.025": 0.29377636317562883, "total_success_rate_ub_0.025": 0.613848760701298, "total_success_rate_lb_0.05": 0.31371987614780983, "total_success_rate_ub_0.05": 0.5979055166048695, "total_success_rate_lb_0.1": 0.3634386435424223, "total_success_rate_ub_0.1": 0.5717472731923613, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "f73c6106c1ced0c2a756bcb15559f52e3dffd065f2811711cf40fb6b82acacbc", "size": 522, "artifact_path": "wandb-client-artifact://bccja055pipajnhixr0buisnbq9tbj3ewykgyumonicipkbeogr5mekg1vb9i8vzy7ax6uyddg94j6ucpoh7atb0yteflrhehfjw9cc5garjtehki1z8t249wqejcxw7/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://2nial4ywy2ej7p5wwajwnuujjye9x1znufny0gr79ihkw5inzzahd80dkkaozvrvq4xd28pi1ezp5i0auznsbq38bwx5yldeqlit59kmshokzte4uto32d4l3yxe2nsr:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_f73c6106c1ced0c2a756.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 297}, "eval/total_success_rate": 0.4571428596973419, "eval/total_success_rate_lb_0.01": 0.2863466441631317, "eval/total_success_rate_ub_0.01": 0.6446045637130737, "eval/total_success_rate_lb_0.025": 0.29377636313438416, "eval/total_success_rate_ub_0.025": 0.6138487458229065, "eval/total_success_rate_lb_0.05": 0.31371986865997314, "eval/total_success_rate_ub_0.05": 0.5979055166244507, "eval/total_success_rate_lb_0.1": 0.3634386360645294, "eval/total_success_rate_ub_0.1": 0.5717472434043884} \ No newline at end of file diff --git a/thh8eztz/wandb/run-20250104_203732-thh8eztz/logs/debug-internal.log b/thh8eztz/wandb/run-20250104_203732-thh8eztz/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..68bb7e94ad223b1faf27a4f75585c57cdfb15332 --- /dev/null +++ b/thh8eztz/wandb/run-20250104_203732-thh8eztz/logs/debug-internal.log @@ -0,0 +1,812 @@ +2025-01-04 20:37:32,036 INFO StreamThr :2013575 [internal.py:wandb_internal():85] W&B internal server running at pid: 2013575, started at: 2025-01-04 20:37:32.036237 +2025-01-04 20:37:32,038 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status +2025-01-04 20:37:32,051 INFO WriterThread:2013575 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/run-thh8eztz.wandb +2025-01-04 20:37:32,051 DEBUG SenderThread:2013575 [sender.py:send():379] send: header +2025-01-04 20:37:32,059 DEBUG SenderThread:2013575 [sender.py:send():379] send: run +2025-01-04 20:37:32,280 INFO SenderThread:2013575 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files +2025-01-04 20:37:32,280 INFO SenderThread:2013575 [sender.py:_start_run_threads():1188] run started: thh8eztz with start time 1736041052.036301 +2025-01-04 20:37:32,301 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 20:37:32,301 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: check_version +2025-01-04 20:37:32,350 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 20:37:32,355 DEBUG HandlerThread:2013575 [system_info.py:__init__():26] System info init +2025-01-04 20:37:32,355 DEBUG HandlerThread:2013575 [system_info.py:__init__():41] System info init done +2025-01-04 20:37:32,355 INFO HandlerThread:2013575 [system_monitor.py:start():194] Starting system monitor +2025-01-04 20:37:32,355 INFO SystemMonitor:2013575 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 20:37:32,355 INFO HandlerThread:2013575 [system_monitor.py:probe():214] Collecting system info +2025-01-04 20:37:32,356 INFO SystemMonitor:2013575 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 20:37:32,356 INFO SystemMonitor:2013575 [interfaces.py:start():188] Started disk monitoring +2025-01-04 20:37:32,357 INFO SystemMonitor:2013575 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 20:37:32,357 INFO SystemMonitor:2013575 [interfaces.py:start():188] Started memory monitoring +2025-01-04 20:37:32,358 INFO SystemMonitor:2013575 [interfaces.py:start():188] Started network monitoring +2025-01-04 20:37:32,368 DEBUG HandlerThread:2013575 [system_info.py:probe():152] Probing system +2025-01-04 20:37:32,370 DEBUG HandlerThread:2013575 [system_info.py:_probe_git():137] Probing git +2025-01-04 20:37:32,375 DEBUG HandlerThread:2013575 [system_info.py:_probe_git():145] Probing git done +2025-01-04 20:37:32,375 DEBUG HandlerThread:2013575 [system_info.py:probe():200] Probing system done +2025-01-04 20:37:32,375 DEBUG HandlerThread:2013575 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T01:37:32.369017', 'startedAt': '2025-01-05T01:37:32.028207', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4124.606125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.984, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.078, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 606.3384628295898}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 20:37:32,375 INFO HandlerThread:2013575 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 20:37:32,375 INFO HandlerThread:2013575 [system_monitor.py:probe():227] Publishing system info +2025-01-04 20:37:32,375 DEBUG HandlerThread:2013575 [system_info.py:_save_code():44] Saving code +2025-01-04 20:37:32,377 DEBUG HandlerThread:2013575 [system_info.py:_save_code():67] Saving code done +2025-01-04 20:37:32,377 DEBUG HandlerThread:2013575 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 20:37:32,438 DEBUG HandlerThread:2013575 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 20:37:32,438 INFO HandlerThread:2013575 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 20:37:32,442 DEBUG SenderThread:2013575 [sender.py:send():379] send: files +2025-01-04 20:37:32,442 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 20:37:32,442 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 20:37:32,442 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 20:37:32,680 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 20:37:32,681 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: python_packages +2025-01-04 20:37:32,682 DEBUG SenderThread:2013575 [sender.py:send():379] send: telemetry +2025-01-04 20:37:32,701 INFO wandb-upload_0:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/trl9szzv-wandb-metadata.json +2025-01-04 20:37:32,702 INFO wandb-upload_1:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/qktoycrx-code/FISH/eval_robot.py +2025-01-04 20:37:32,725 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:37:32,725 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:32,725 INFO HandlerThread:2013575 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/tb" + save: true +} +control { + relay_id: "46f399648cfc4418b20aca9cdc6e71fe" +} +_info { + stream_id: "thh8eztz" +} + +2025-01-04 20:37:32,725 DEBUG HandlerThread:2013575 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 20:37:32,726 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:37:32,735 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/vjeler6c-diff.patch +2025-01-04 20:37:32,839 DEBUG SenderThread:2013575 [sender.py:send():379] send: tbrecord +2025-01-04 20:37:32,840 DEBUG SenderThread:2013575 [sender.py:send():379] send: files +2025-01-04 20:37:32,840 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 with policy live +2025-01-04 20:37:33,027 INFO wandb-upload_0:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/njydtob1-events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:37:33,281 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/diff.patch +2025-01-04 20:37:33,281 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:37:33,282 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/requirements.txt +2025-01-04 20:37:33,282 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:37:33,282 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/code/FISH/eval_robot.py +2025-01-04 20:37:33,282 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-metadata.json +2025-01-04 20:37:33,282 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/code/FISH +2025-01-04 20:37:33,282 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/code +2025-01-04 20:37:33,681 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:34,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:35,280 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:37:35,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:36,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:37,681 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:37:37,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:38,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:39,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:40,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:41,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:42,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:37:42,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:43,683 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:44,683 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:45,683 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:46,683 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:47,681 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:37:47,681 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:37:47,721 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:47,799 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:37:48,683 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:49,466 DEBUG SenderThread:2013575 [sender.py:send():379] send: config +2025-01-04 20:37:49,683 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:50,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:51,285 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:37:51,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:52,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:53,189 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:37:53,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:54,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:55,286 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:37:55,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:56,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:57,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:58,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:37:58,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:37:59,287 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:37:59,685 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:00,685 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:01,288 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:38:01,685 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:02,681 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:38:02,681 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:38:02,725 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:03,289 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:38:03,686 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:03,739 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:04,289 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/config.yaml +2025-01-04 20:38:04,686 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:05,289 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:38:05,686 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:06,686 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:07,687 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:08,687 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:09,683 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:09,687 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:10,687 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:11,687 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:12,687 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:13,688 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:14,683 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:14,688 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:15,688 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:16,688 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:17,681 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:38:17,681 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:38:17,725 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:18,688 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:19,688 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:19,753 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:20,689 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:21,689 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:22,689 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:23,689 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:24,689 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:25,683 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:25,689 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:26,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:27,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:28,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:29,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:30,684 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:30,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:31,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:32,358 DEBUG SystemMonitor:2013575 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 20:38:32,359 DEBUG SenderThread:2013575 [sender.py:send():379] send: stats +2025-01-04 20:38:32,681 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:38:32,681 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:38:32,725 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:33,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:34,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:35,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:35,830 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:36,691 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:37,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:38,693 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:39,693 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:40,455 DEBUG SenderThread:2013575 [sender.py:send():379] send: files +2025-01-04 20:38:40,455 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_04d1dff61e6bb2dac431.mp4 with policy now +2025-01-04 20:38:40,455 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:38:40,456 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:38:40,456 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:38:40,457 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:38:40,694 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:40,741 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/ujd7fqor-media/videos/eval/0_eval_0_04d1dff61e6bb2dac431.mp4 +2025-01-04 20:38:41,298 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:38:41,298 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/0_eval_0_04d1dff61e6bb2dac431.mp4 +2025-01-04 20:38:41,299 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:38:41,299 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos +2025-01-04 20:38:41,299 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval +2025-01-04 20:38:41,299 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media +2025-01-04 20:38:41,472 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:41,591 INFO wandb-upload_0:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/oi317r9a-media/videos/eval/0_eval_0_04d1dff61e6bb2dac431.mp4 +2025-01-04 20:38:41,694 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:42,694 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:43,299 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:38:43,694 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:44,694 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:45,300 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:38:45,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:46,585 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:46,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:47,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:38:47,682 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:38:47,725 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:48,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:49,301 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:38:49,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:50,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:51,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:51,836 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:52,696 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:53,696 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:54,696 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:55,696 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:56,696 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:57,685 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:38:57,697 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:58,697 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:38:59,701 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:00,697 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:01,697 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:02,359 DEBUG SenderThread:2013575 [sender.py:send():379] send: stats +2025-01-04 20:39:02,681 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:39:02,682 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:39:02,729 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:02,797 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:03,697 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:04,697 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:05,698 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:06,698 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:07,737 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:08,307 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:39:08,547 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/e2wwpkkb-events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:39:08,698 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:08,713 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:09,307 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:39:09,698 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:10,704 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:11,308 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:39:11,700 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:12,701 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:13,309 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:39:13,700 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:13,807 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:14,700 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:15,310 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:39:15,700 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:16,700 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:17,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:39:17,682 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:39:17,725 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:18,701 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:19,701 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:19,739 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:20,701 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:21,701 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:22,701 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:23,701 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:24,702 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:25,686 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:25,702 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:26,702 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:27,702 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:28,702 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:29,702 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:30,702 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:31,686 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:31,703 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:32,360 DEBUG SenderThread:2013575 [sender.py:send():379] send: stats +2025-01-04 20:39:32,682 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:39:32,682 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:39:32,725 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:33,703 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:34,703 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:35,703 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:36,703 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:36,808 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:37,703 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:38,703 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:39,648 DEBUG SenderThread:2013575 [sender.py:send():379] send: files +2025-01-04 20:39:39,648 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_852c010e6bb3816412aa.mp4 with policy now +2025-01-04 20:39:39,654 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:39:39,654 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:39:39,654 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:39:39,655 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:39:39,703 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:39,908 INFO wandb-upload_0:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/mzyo1grh-media/videos/eval/1_eval_1_852c010e6bb3816412aa.mp4 +2025-01-04 20:39:40,317 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:39:40,317 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/1_eval_1_852c010e6bb3816412aa.mp4 +2025-01-04 20:39:40,318 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval +2025-01-04 20:39:40,586 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/0c4fxa9w-media/videos/eval/1_eval_1_852c010e6bb3816412aa.mp4 +2025-01-04 20:39:40,704 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:41,318 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:39:41,708 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:42,677 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:42,708 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:43,709 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:44,708 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:45,319 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:39:45,711 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:46,709 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:47,320 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:39:47,685 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:39:47,685 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:39:47,725 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:47,748 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:48,708 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:49,711 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:50,713 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:51,710 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:52,712 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:53,691 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:53,718 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:54,713 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:55,711 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:56,717 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:57,717 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:58,718 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:39:59,690 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:39:59,724 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:00,723 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:01,720 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:02,361 DEBUG SenderThread:2013575 [sender.py:send():379] send: stats +2025-01-04 20:40:02,686 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:40:02,686 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:40:02,729 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:03,724 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:04,720 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:04,738 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:05,723 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:06,724 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:07,722 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:08,723 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:09,722 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:10,691 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:10,727 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:11,728 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:12,726 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:13,196 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:40:13,196 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:40:13,196 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:40:13,328 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:40:13,328 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:40:13,563 INFO wandb-upload_0:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/p4b8hcom-events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:40:13,727 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:14,731 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:15,329 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:40:15,735 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:16,377 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:16,727 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:17,329 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:40:17,694 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:40:17,694 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:40:17,730 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:18,729 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:19,330 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:40:19,730 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:20,731 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:21,731 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:21,792 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:22,731 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:23,735 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:24,733 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:25,733 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:26,735 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:27,333 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:40:27,489 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:27,734 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:28,736 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:29,734 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:30,735 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:31,735 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:32,361 DEBUG SenderThread:2013575 [sender.py:send():379] send: stats +2025-01-04 20:40:32,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:40:32,695 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:40:32,741 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:32,766 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:33,335 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:40:33,737 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:34,736 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:35,735 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:36,739 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:37,164 DEBUG SenderThread:2013575 [sender.py:send():379] send: files +2025-01-04 20:40:37,164 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_10e0e64153d3c9c65e4d.mp4 with policy now +2025-01-04 20:40:37,165 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:40:37,166 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:40:37,166 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:40:37,166 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:40:37,336 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:40:37,337 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:40:37,337 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/2_eval_2_10e0e64153d3c9c65e4d.mp4 +2025-01-04 20:40:37,337 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval +2025-01-04 20:40:37,424 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/tutyof3y-media/videos/eval/2_eval_2_10e0e64153d3c9c65e4d.mp4 +2025-01-04 20:40:37,687 INFO wandb-upload_0:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/qb6w7ex6-media/videos/eval/2_eval_2_10e0e64153d3c9c65e4d.mp4 +2025-01-04 20:40:37,740 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:38,699 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:38,743 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:39,337 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:40:39,738 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:40,765 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:41,741 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:42,741 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:43,338 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:40:43,738 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:44,549 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:44,742 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:45,339 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:40:45,745 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:46,748 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:47,694 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:40:47,695 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:40:47,752 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:48,749 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:49,745 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:49,757 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:50,749 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:51,756 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:52,748 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:53,755 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:54,751 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:55,696 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:40:55,758 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:56,757 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:57,757 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:58,759 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:40:59,756 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:00,025 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:41:00,026 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:41:00,026 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:41:00,344 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:41:00,344 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:41:00,766 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:01,344 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:01,593 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:01,769 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:02,362 DEBUG SenderThread:2013575 [sender.py:send():379] send: stats +2025-01-04 20:41:02,696 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:41:02,697 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:41:02,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:03,345 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:03,768 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:04,769 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:05,346 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:05,765 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:06,765 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:07,224 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:07,346 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:07,765 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:08,766 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:09,766 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:10,766 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:11,785 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:12,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:12,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:13,348 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:13,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:14,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:15,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:16,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:17,694 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:41:17,694 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:41:17,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:17,826 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:18,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:19,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:20,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:21,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:22,767 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:23,696 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:23,768 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:24,768 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:25,809 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:26,785 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:26,869 DEBUG SenderThread:2013575 [sender.py:send():379] send: files +2025-01-04 20:41:26,869 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_0730c18e4ed0ae46c605.mp4 with policy now +2025-01-04 20:41:26,872 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:41:26,872 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:41:26,872 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:41:26,873 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:41:27,131 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/1ffl4gt0-media/videos/eval/3_eval_3_0730c18e4ed0ae46c605.mp4 +2025-01-04 20:41:27,353 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:27,353 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:41:27,353 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/3_eval_3_0730c18e4ed0ae46c605.mp4 +2025-01-04 20:41:27,354 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval +2025-01-04 20:41:27,647 INFO wandb-upload_0:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/cyjt0a22-media/videos/eval/3_eval_3_0730c18e4ed0ae46c605.mp4 +2025-01-04 20:41:27,771 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:28,697 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:28,775 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:29,354 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:29,784 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:30,785 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:31,355 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:31,784 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:32,362 DEBUG SenderThread:2013575 [sender.py:send():379] send: stats +2025-01-04 20:41:32,700 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:41:32,701 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:41:32,788 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:33,790 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:34,177 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:34,785 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:35,356 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:35,790 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:36,787 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:37,070 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:41:37,070 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:41:37,070 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:41:37,357 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:41:37,357 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:41:37,593 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/8ds8qg2u-events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:41:37,787 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:38,795 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:39,357 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:39,796 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:39,853 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:40,790 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:41,358 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:41,788 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:42,788 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:43,359 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:43,788 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:44,788 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:45,696 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:45,788 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:46,788 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:47,695 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:41:47,695 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:41:47,788 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:48,789 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:49,789 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:50,789 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:50,834 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:51,789 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:52,789 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:53,362 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:53,789 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:54,789 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:55,790 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:56,615 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:41:56,790 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:57,364 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:57,644 DEBUG SenderThread:2013575 [sender.py:send():379] send: files +2025-01-04 20:41:57,644 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_50b70b008ecfe74e7155.mp4 with policy now +2025-01-04 20:41:57,650 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:41:57,650 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:41:57,650 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:41:57,650 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:41:57,790 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:57,889 INFO wandb-upload_0:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/6ir135qj-media/videos/eval/4_eval_4_50b70b008ecfe74e7155.mp4 +2025-01-04 20:41:58,364 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:41:58,364 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/4_eval_4_50b70b008ecfe74e7155.mp4 +2025-01-04 20:41:58,364 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval +2025-01-04 20:41:58,625 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/xbl5egrt-media/videos/eval/4_eval_4_50b70b008ecfe74e7155.mp4 +2025-01-04 20:41:58,799 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:41:59,364 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:41:59,804 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:00,805 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:01,809 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:01,933 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:42:02,363 DEBUG SenderThread:2013575 [sender.py:send():379] send: stats +2025-01-04 20:42:02,707 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:42:02,707 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:42:02,812 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:03,366 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:42:03,808 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:04,816 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:05,366 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:42:05,811 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:06,824 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:07,706 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:42:07,810 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:08,817 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:09,814 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:10,814 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:11,820 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:12,707 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:42:12,818 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:13,820 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:14,820 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:15,825 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:16,822 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:17,708 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:42:17,715 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:42:17,715 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:42:17,828 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:18,831 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:19,828 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:20,126 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:42:20,126 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:20,126 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:20,371 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:42:20,372 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:42:20,556 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 20:42:20,556 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 20:42:20,754 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:42:20,823 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:21,021 INFO wandb-upload_0:2013575 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp3y2jujh2 +2025-01-04 20:42:21,162 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 20:42:21,172 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:42:21,372 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:42:21,372 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:42:21,372 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/table/eval/total_success_rate_ci_6_f73c6106c1ced0c2a756.table.json +2025-01-04 20:42:21,372 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 20:42:21,372 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media +2025-01-04 20:42:21,372 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/table +2025-01-04 20:42:21,373 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/table/eval +2025-01-04 20:42:21,398 INFO SenderThread:2013575 [sender.py:send_request_log_artifact():1525] logged artifact run-thh8eztz-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTExMTQ0Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MDIyMg==', 'latestArtifact': None}} +2025-01-04 20:42:21,398 DEBUG SenderThread:2013575 [sender.py:send():379] send: files +2025-01-04 20:42:21,398 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json with policy now +2025-01-04 20:42:21,398 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:42:21,400 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,402 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,402 DEBUG SenderThread:2013575 [sender.py:send():379] send: telemetry +2025-01-04 20:42:21,402 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,402 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,402 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,403 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,403 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,403 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,403 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,404 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,404 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,404 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,405 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,405 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,405 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,405 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,405 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,406 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,406 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,406 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,406 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,407 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,407 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:21,407 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:21,408 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 20:42:21,583 INFO wandb-upload_1:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/5ri5di85-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 20:42:21,616 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/7f1q6zll-media/table/eval/total_success_rate_ci_6_f73c6106c1ced0c2a756.table.json +2025-01-04 20:42:21,758 INFO wandb-upload_0:2013575 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpomz4jqfw +2025-01-04 20:42:21,818 INFO wandb-upload_1:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/hwwdg8aj-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 20:42:21,823 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:22,143 INFO SenderThread:2013575 [sender.py:send_request_log_artifact():1525] logged artifact run-thh8eztz-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTExMTQ1OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MDIyNg==', 'latestArtifact': None}} +2025-01-04 20:42:22,143 DEBUG SenderThread:2013575 [sender.py:send():379] send: files +2025-01-04 20:42:22,143 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_f73c6106c1ced0c2a756.table.json with policy now +2025-01-04 20:42:22,143 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:42:22,143 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:22,144 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:22,372 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:42:22,375 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/u2xg3bu1-media/table/eval/total_success_rate_ci_6_f73c6106c1ced0c2a756.table.json +2025-01-04 20:42:22,824 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:23,373 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:42:23,701 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:42:23,824 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:24,824 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:25,824 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:26,824 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:27,824 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:28,824 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:29,705 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:42:29,825 DEBUG SenderThread:2013575 [sender.py:send():379] send: exit +2025-01-04 20:42:29,825 INFO SenderThread:2013575 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 20:42:29,825 INFO SenderThread:2013575 [sender.py:send_exit():588] handling runtime: 297 +2025-01-04 20:42:29,826 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:29,827 INFO SenderThread:2013575 [sender.py:send_exit():594] send defer +2025-01-04 20:42:29,827 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:29,827 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 20:42:29,827 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:29,827 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 20:42:29,827 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 1 +2025-01-04 20:42:29,827 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:29,827 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 20:42:29,827 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:29,827 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 20:42:29,827 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 2 +2025-01-04 20:42:29,827 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:29,827 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 20:42:29,827 INFO HandlerThread:2013575 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 20:42:29,828 INFO HandlerThread:2013575 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 20:42:29,828 DEBUG SystemMonitor:2013575 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 20:42:29,828 INFO HandlerThread:2013575 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 20:42:29,828 DEBUG SystemMonitor:2013575 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 20:42:29,885 INFO HandlerThread:2013575 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 20:42:29,885 INFO HandlerThread:2013575 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 20:42:29,886 INFO HandlerThread:2013575 [interfaces.py:finish():200] Joined network monitor +2025-01-04 20:42:29,886 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:29,886 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 20:42:29,886 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 3 +2025-01-04 20:42:29,886 DEBUG SenderThread:2013575 [sender.py:send():379] send: stats +2025-01-04 20:42:29,886 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:29,886 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 20:42:29,887 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:29,887 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 20:42:29,887 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 4 +2025-01-04 20:42:29,887 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:29,887 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 20:42:30,375 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/config.yaml +2025-01-04 20:42:30,376 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:42:31,376 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:42:36,126 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:42:36,126 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:42:36,126 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 20:42:36,127 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:36,127 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 20:42:36,127 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 5 +2025-01-04 20:42:36,127 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:42:36,127 DEBUG SenderThread:2013575 [sender.py:send():379] send: history +2025-01-04 20:42:36,127 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:42:36,128 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:36,128 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 20:42:36,128 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:36,129 DEBUG SenderThread:2013575 [sender.py:send():379] send: summary +2025-01-04 20:42:36,132 INFO SenderThread:2013575 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:42:36,133 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:36,133 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 20:42:36,133 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 6 +2025-01-04 20:42:36,133 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:36,133 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 20:42:36,133 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:36,133 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 20:42:36,133 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 7 +2025-01-04 20:42:36,133 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:42:36,133 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:36,133 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 20:42:36,134 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:36,134 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 20:42:36,378 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:42:36,826 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:42:38,952 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 8 +2025-01-04 20:42:38,952 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:42:38,952 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:38,952 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 20:42:38,952 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:38,952 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 20:42:38,952 INFO SenderThread:2013575 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 20:42:38,955 INFO SenderThread:2013575 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 20:42:38,957 INFO SenderThread:2013575 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 20:42:38,969 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 9 +2025-01-04 20:42:38,969 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:38,969 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 20:42:38,969 DEBUG SenderThread:2013575 [sender.py:send():379] send: artifact +2025-01-04 20:42:39,303 INFO wandb-upload_2:2013575 [upload_job.py:push():85] Skipped uploading /tmp/tmp0m470vzd/wandb-job.json +2025-01-04 20:42:39,304 INFO wandb-upload_0:2013575 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpobqrfexr +2025-01-04 20:42:39,379 INFO Thread-12 :2013575 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:42:39,432 INFO wandb-upload_1:2013575 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_0vhxu1f +2025-01-04 20:42:39,827 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:42:39,956 INFO SenderThread:2013575 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTExMTc3MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEwMTI1NQ==', 'versionIndex': 416}}} +2025-01-04 20:42:39,956 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:39,956 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 20:42:39,956 INFO SenderThread:2013575 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 20:42:40,380 INFO SenderThread:2013575 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files +2025-01-04 20:42:40,380 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/config.yaml config.yaml +2025-01-04 20:42:40,380 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json wandb-summary.json +2025-01-04 20:42:40,380 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/requirements.txt requirements.txt +2025-01-04 20:42:40,380 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-metadata.json wandb-metadata.json +2025-01-04 20:42:40,381 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:42:40,381 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log output.log +2025-01-04 20:42:40,384 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/diff.patch diff.patch +2025-01-04 20:42:40,386 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 20:42:40,389 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/2_eval_2_10e0e64153d3c9c65e4d.mp4 media/videos/eval/2_eval_2_10e0e64153d3c9c65e4d.mp4 +2025-01-04 20:42:40,389 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/3_eval_3_0730c18e4ed0ae46c605.mp4 media/videos/eval/3_eval_3_0730c18e4ed0ae46c605.mp4 +2025-01-04 20:42:40,389 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/0_eval_0_04d1dff61e6bb2dac431.mp4 media/videos/eval/0_eval_0_04d1dff61e6bb2dac431.mp4 +2025-01-04 20:42:40,389 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/1_eval_1_852c010e6bb3816412aa.mp4 media/videos/eval/1_eval_1_852c010e6bb3816412aa.mp4 +2025-01-04 20:42:40,389 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/videos/eval/4_eval_4_50b70b008ecfe74e7155.mp4 media/videos/eval/4_eval_4_50b70b008ecfe74e7155.mp4 +2025-01-04 20:42:40,389 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-04 20:42:40,389 INFO SenderThread:2013575 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/media/table/eval/total_success_rate_ci_6_f73c6106c1ced0c2a756.table.json media/table/eval/total_success_rate_ci_6_f73c6106c1ced0c2a756.table.json +2025-01-04 20:42:40,389 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 10 +2025-01-04 20:42:40,389 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:42:40,390 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:40,390 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 20:42:40,390 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:40,390 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 20:42:40,390 INFO SenderThread:2013575 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 20:42:40,570 INFO wandb-upload_2:2013575 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/config.yaml +2025-01-04 20:42:40,574 INFO wandb-upload_1:2013575 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/requirements.txt +2025-01-04 20:42:40,598 INFO wandb-upload_0:2013575 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/wandb-summary.json +2025-01-04 20:42:40,604 INFO wandb-upload_4:2013575 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/files/output.log +2025-01-04 20:42:40,664 INFO wandb-upload_3:2013575 [upload_job.py:push():130] Uploaded file /tmp/tmp0kw43sgxwandb/1j0f0tqh-events.out.tfevents.1736041052.leonmkim-ROG-Strix-G15CS-G15CS.2013380.0 +2025-01-04 20:42:40,828 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:42:40,828 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:42:40,864 INFO Thread-11 (_thread_body):2013575 [sender.py:transition_state():614] send defer: 11 +2025-01-04 20:42:40,864 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:40,864 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 20:42:40,865 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:40,865 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 20:42:40,865 INFO SenderThread:2013575 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 20:42:40,866 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 12 +2025-01-04 20:42:40,866 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:40,866 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 20:42:40,866 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:40,866 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 20:42:40,866 INFO SenderThread:2013575 [file_stream.py:finish():601] file stream finish called +2025-01-04 20:42:41,136 INFO SenderThread:2013575 [file_stream.py:finish():605] file stream finish is done +2025-01-04 20:42:41,136 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 13 +2025-01-04 20:42:41,136 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:41,136 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 20:42:41,137 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:41,137 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 20:42:41,137 INFO SenderThread:2013575 [sender.py:transition_state():614] send defer: 14 +2025-01-04 20:42:41,137 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:42:41,137 INFO HandlerThread:2013575 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 20:42:41,137 DEBUG SenderThread:2013575 [sender.py:send():379] send: final +2025-01-04 20:42:41,137 DEBUG SenderThread:2013575 [sender.py:send():379] send: footer +2025-01-04 20:42:41,137 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: defer +2025-01-04 20:42:41,137 INFO SenderThread:2013575 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 20:42:41,138 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:42:41,138 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:42:41,138 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 20:42:41,139 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:42:41,139 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 20:42:41,139 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: server_info +2025-01-04 20:42:41,141 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 20:42:41,141 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:42:41,202 DEBUG SenderThread:2013575 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:42:41,202 INFO MainThread:2013575 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 20:42:41,203 INFO MainThread:2013575 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 20:42:41,203 INFO MainThread:2013575 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 20:42:41,203 DEBUG HandlerThread:2013575 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 20:42:41,203 INFO HandlerThread:2013575 [handler.py:finish():882] shutting down handler +2025-01-04 20:42:42,139 INFO WriterThread:2013575 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/run-thh8eztz.wandb +2025-01-04 20:42:42,202 INFO SenderThread:2013575 [sender.py:finish():1615] shutting down sender +2025-01-04 20:42:42,203 INFO SenderThread:2013575 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 20:42:42,203 INFO SenderThread:2013575 [file_pusher.py:join():175] waiting for file pusher diff --git a/thh8eztz/wandb/run-20250104_203732-thh8eztz/logs/debug.log b/thh8eztz/wandb/run-20250104_203732-thh8eztz/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..5b925fbf2e628de7cafcdb1ad244ff1c6d58a646 --- /dev/null +++ b/thh8eztz/wandb/run-20250104_203732-thh8eztz/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 20:37:32,030 INFO MainThread:2013380 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 20:37:32,030 INFO MainThread:2013380 [wandb_setup.py:_flush():76] Configure stats pid to 2013380 +2025-01-04 20:37:32,030 INFO MainThread:2013380 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 20:37:32,030 INFO MainThread:2013380 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/settings +2025-01-04 20:37:32,030 INFO MainThread:2013380 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 20:37:32,030 INFO MainThread:2013380 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 20:37:32,030 INFO MainThread:2013380 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 20:37:32,030 INFO MainThread:2013380 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 20:37:32,031 INFO MainThread:2013380 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/logs/debug.log +2025-01-04 20:37:32,031 INFO MainThread:2013380 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/wandb/run-20250104_203732-thh8eztz/logs/debug-internal.log +2025-01-04 20:37:32,031 INFO MainThread:2013380 [wandb_init.py:init():569] calling init triggers +2025-01-04 20:37:32,031 INFO MainThread:2013380 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz'} +2025-01-04 20:37:32,031 INFO MainThread:2013380 [wandb_init.py:init():619] starting backend +2025-01-04 20:37:32,031 INFO MainThread:2013380 [wandb_init.py:init():623] setting up manager +2025-01-04 20:37:32,035 INFO MainThread:2013380 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 20:37:32,036 INFO MainThread:2013380 [wandb_init.py:init():631] backend started and connected +2025-01-04 20:37:32,050 INFO MainThread:2013380 [wandb_init.py:init():720] updated telemetry +2025-01-04 20:37:32,057 INFO MainThread:2013380 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 20:37:32,300 INFO MainThread:2013380 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 20:37:32,339 INFO MainThread:2013380 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 20:37:32,339 INFO MainThread:2013380 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 20:37:32,681 INFO MainThread:2013380 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 20:37:32,681 INFO MainThread:2013380 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 20:37:32,681 INFO MainThread:2013380 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 20:37:32,681 INFO MainThread:2013380 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 20:37:32,683 INFO MainThread:2013380 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 20:37:32,684 INFO MainThread:2013380 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/thh8eztz/tb, True +2025-01-04 20:37:49,466 INFO MainThread:2013380 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-04 20:42:42,204 WARNING MsgRouterThr:2013380 [router.py:message_loop():77] message_loop has been closed diff --git a/thh8eztz/wandb/run-20250104_203732-thh8eztz/run-thh8eztz.wandb b/thh8eztz/wandb/run-20250104_203732-thh8eztz/run-thh8eztz.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d96a7729065b1cf72e0af3dab9e3ed7564a9114b Binary files /dev/null and b/thh8eztz/wandb/run-20250104_203732-thh8eztz/run-thh8eztz.wandb differ diff --git a/tltaigwz/.hydra/config.yaml b/tltaigwz/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/tltaigwz/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/tltaigwz/.hydra/hydra.yaml b/tltaigwz/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54ed8efc97851cb1185b056d317be8b9dbd1c4af --- /dev/null +++ b/tltaigwz/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/tltaigwz + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/tltaigwz/.hydra/overrides.yaml b/tltaigwz/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/tltaigwz/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/tltaigwz/eval_policy.log b/tltaigwz/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..46a32fb06e4f8185c593f481481c7a71b39036f7 --- /dev/null +++ b/tltaigwz/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-03 20:25:42,569][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 20:25:42,572][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-03 20:25:46,899][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/ts8zycr2/.hydra/config.yaml b/ts8zycr2/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/ts8zycr2/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/ts8zycr2/.hydra/hydra.yaml b/ts8zycr2/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8af49de04d0a8c0b1278788824b77b7a3f0e149c --- /dev/null +++ b/ts8zycr2/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/ts8zycr2/.hydra/overrides.yaml b/ts8zycr2/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/ts8zycr2/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/ts8zycr2/episode_rosbags/aligned_depth_to_color_K.npy b/ts8zycr2/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/ts8zycr2/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/ts8zycr2/episode_rosbags/cam_tf_world.npy b/ts8zycr2/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/ts8zycr2/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/ts8zycr2/episode_rosbags/color_K.npy b/ts8zycr2/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/ts8zycr2/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/ts8zycr2/episode_rosbags/depth_K.npy b/ts8zycr2/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/ts8zycr2/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/ts8zycr2/episode_rosbags/episode_4_2025-01-03-20-43-26.bag b/ts8zycr2/episode_rosbags/episode_4_2025-01-03-20-43-26.bag new file mode 100644 index 0000000000000000000000000000000000000000..0e45552d07809aacbe98af76f782e5865079afce --- /dev/null +++ b/ts8zycr2/episode_rosbags/episode_4_2025-01-03-20-43-26.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7074d831b1791ed3de0b5d22f5f52e0e7a3fd334ded73ad00c3a0724df3e3350 +size 2076201296 diff --git a/ts8zycr2/eval_robot.log b/ts8zycr2/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..fb92e9dbfb4d7283a4ebe74eb31f499342261881 --- /dev/null +++ b/ts8zycr2/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 20:39:14,668][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 20:39:14,671][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/ts8zycr2/eval_video/0_eval.mp4 b/ts8zycr2/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b1e22574edf752b88ab5cfe0248de102d163d93f --- /dev/null +++ b/ts8zycr2/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3881364072b3e77caced7bb7be9deee5531f0ea3c1eb402de1f4ffb612b0542e +size 1439292 diff --git a/ts8zycr2/eval_video/1_eval.mp4 b/ts8zycr2/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0ada39852e82c62e7e335828b5f9a29a09608129 --- /dev/null +++ b/ts8zycr2/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f514b24221df96e8edb88f9e6195cfee832325974feead82d73898d6c40986b +size 1456453 diff --git a/ts8zycr2/eval_video/2_eval.mp4 b/ts8zycr2/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..37c6425cf5e44b061d272116c1fdc570b5d8d87b --- /dev/null +++ b/ts8zycr2/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebd3ab9d4a47a599597242acc20388d3e1e4d82913dbccdeab479f251cd0fba2 +size 1451500 diff --git a/ts8zycr2/eval_video/3_eval.mp4 b/ts8zycr2/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5729eec17b708af7947b14d2551817768af111d4 --- /dev/null +++ b/ts8zycr2/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caa22dfc15ca57c8e31dfaac858c43b2cddf3383e4b320501edb19c04f674bf5 +size 1467645 diff --git a/ts8zycr2/eval_video/4_eval.mp4 b/ts8zycr2/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fe53b6f340fa58ba6f48ef5f2222de1431496bea --- /dev/null +++ b/ts8zycr2/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba85108b8a446815d41f910e718c6f3d1fd0df3e53ccb6ead00253d8c730b7d1 +size 1310145 diff --git a/ts8zycr2/tb/events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 b/ts8zycr2/tb/events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 new file mode 100644 index 0000000000000000000000000000000000000000..c3de6292552d05efe211fe23d6cec4928ba64b68 --- /dev/null +++ b/ts8zycr2/tb/events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8258289ff44c788757d47dbab713ceb2cf05bdd797ed167c2e4312093f7e42b9 +size 6539 diff --git a/ts8zycr2/wandb/debug-internal.log b/ts8zycr2/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1e9aca7076fb72087626b4cd883253b62cf5ed2a --- /dev/null +++ b/ts8zycr2/wandb/debug-internal.log @@ -0,0 +1,831 @@ +2025-01-03 20:39:20,345 INFO StreamThr :1968165 [internal.py:wandb_internal():85] W&B internal server running at pid: 1968165, started at: 2025-01-03 20:39:20.345539 +2025-01-03 20:39:20,346 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status +2025-01-03 20:39:20,359 INFO WriterThread:1968165 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/run-ts8zycr2.wandb +2025-01-03 20:39:20,359 DEBUG SenderThread:1968165 [sender.py:send():379] send: header +2025-01-03 20:39:20,367 DEBUG SenderThread:1968165 [sender.py:send():379] send: run +2025-01-03 20:39:20,649 INFO SenderThread:1968165 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files +2025-01-03 20:39:20,649 INFO SenderThread:1968165 [sender.py:_start_run_threads():1188] run started: ts8zycr2 with start time 1735954760.346241 +2025-01-03 20:39:20,667 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 20:39:20,667 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: check_version +2025-01-03 20:39:20,742 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 20:39:20,746 DEBUG HandlerThread:1968165 [system_info.py:__init__():26] System info init +2025-01-03 20:39:20,747 DEBUG HandlerThread:1968165 [system_info.py:__init__():41] System info init done +2025-01-03 20:39:20,747 INFO HandlerThread:1968165 [system_monitor.py:start():194] Starting system monitor +2025-01-03 20:39:20,747 INFO SystemMonitor:1968165 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 20:39:20,747 INFO HandlerThread:1968165 [system_monitor.py:probe():214] Collecting system info +2025-01-03 20:39:20,747 INFO SystemMonitor:1968165 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 20:39:20,748 INFO SystemMonitor:1968165 [interfaces.py:start():188] Started disk monitoring +2025-01-03 20:39:20,749 INFO SystemMonitor:1968165 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 20:39:20,749 INFO SystemMonitor:1968165 [interfaces.py:start():188] Started memory monitoring +2025-01-03 20:39:20,750 INFO SystemMonitor:1968165 [interfaces.py:start():188] Started network monitoring +2025-01-03 20:39:20,760 DEBUG HandlerThread:1968165 [system_info.py:probe():152] Probing system +2025-01-03 20:39:20,761 DEBUG HandlerThread:1968165 [system_info.py:_probe_git():137] Probing git +2025-01-03 20:39:20,766 DEBUG HandlerThread:1968165 [system_info.py:_probe_git():145] Probing git done +2025-01-03 20:39:20,766 DEBUG HandlerThread:1968165 [system_info.py:probe():200] Probing system done +2025-01-03 20:39:20,766 DEBUG HandlerThread:1968165 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T01:39:20.760201', 'startedAt': '2025-01-04T01:39:20.338752', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3943.6670000000004, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4500.25, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4511.948, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4505.875, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4501.688, 'min': 800.0, 'max': 4700.0}, {'current': 4502.934, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 746.461067199707}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 20:39:20,766 INFO HandlerThread:1968165 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 20:39:20,766 INFO HandlerThread:1968165 [system_monitor.py:probe():227] Publishing system info +2025-01-03 20:39:20,766 DEBUG HandlerThread:1968165 [system_info.py:_save_code():44] Saving code +2025-01-03 20:39:20,768 DEBUG HandlerThread:1968165 [system_info.py:_save_code():67] Saving code done +2025-01-03 20:39:20,768 DEBUG HandlerThread:1968165 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 20:39:20,831 DEBUG HandlerThread:1968165 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 20:39:20,831 INFO HandlerThread:1968165 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 20:39:20,835 DEBUG SenderThread:1968165 [sender.py:send():379] send: files +2025-01-03 20:39:20,835 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 20:39:20,835 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 20:39:20,835 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 20:39:21,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 20:39:21,087 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: python_packages +2025-01-03 20:39:21,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:21,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:39:21,089 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/zdme6q0t-wandb-metadata.json +2025-01-03 20:39:21,089 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:39:21,098 INFO wandb-upload_2:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/gdd2ad18-diff.patch +2025-01-03 20:39:21,106 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/q7txf2dp-code/FISH/eval_robot.py +2025-01-03 20:39:21,129 INFO HandlerThread:1968165 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/tb" + save: true +} +control { + relay_id: "e1fa12757c664fe9b3563e6e097209d1" +} +_info { + stream_id: "ts8zycr2" +} + +2025-01-03 20:39:21,130 DEBUG HandlerThread:1968165 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 20:39:21,231 DEBUG SenderThread:1968165 [sender.py:send():379] send: telemetry +2025-01-03 20:39:21,231 DEBUG SenderThread:1968165 [sender.py:send():379] send: tbrecord +2025-01-03 20:39:21,232 DEBUG SenderThread:1968165 [sender.py:send():379] send: files +2025-01-03 20:39:21,232 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 with policy live +2025-01-03 20:39:21,416 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/av6ooeil-events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:39:21,651 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-metadata.json +2025-01-03 20:39:21,651 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/requirements.txt +2025-01-03 20:39:21,651 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/code/FISH/eval_robot.py +2025-01-03 20:39:21,651 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/diff.patch +2025-01-03 20:39:21,652 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:39:21,652 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:39:21,652 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/code +2025-01-03 20:39:21,652 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/code/FISH +2025-01-03 20:39:22,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:23,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:23,650 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:39:24,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:25,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:25,129 DEBUG SenderThread:1968165 [sender.py:send():379] send: config +2025-01-03 20:39:26,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:39:26,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:27,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:27,651 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:39:28,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:29,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:30,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:31,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:39:31,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:31,653 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:39:32,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:33,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:33,653 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:39:34,089 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:35,089 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:36,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:39:36,087 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:39:36,129 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:36,209 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:39:37,089 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:38,089 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:39,089 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:40,089 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:41,089 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:41,655 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:39:42,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:39:42,089 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:43,091 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:44,090 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:45,090 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:45,656 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:39:46,090 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:47,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:39:47,090 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:48,090 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:49,091 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:50,091 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:51,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:39:51,087 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:39:51,133 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:52,091 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:52,205 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:39:52,658 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/config.yaml +2025-01-03 20:39:53,093 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:54,093 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:55,093 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:56,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:57,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:58,090 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:39:58,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:59,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:00,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:01,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:02,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:03,090 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:03,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:04,095 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:05,095 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:06,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:40:06,087 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:40:06,129 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:07,095 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:08,095 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:08,188 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:09,095 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:10,095 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:11,095 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:12,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:13,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:13,425 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:13,664 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:40:14,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:15,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:16,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:17,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:18,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:18,511 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:18,522 DEBUG SenderThread:1968165 [sender.py:send():379] send: files +2025-01-03 20:40:18,522 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_3881364072b3e77caced.mp4 with policy now +2025-01-03 20:40:18,522 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:40:18,523 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:40:18,523 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:40:18,524 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:40:18,665 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:40:18,665 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/0_eval_0_3881364072b3e77caced.mp4 +2025-01-03 20:40:18,666 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval +2025-01-03 20:40:18,666 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos +2025-01-03 20:40:18,666 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media +2025-01-03 20:40:18,790 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/1qeg35c1-media/videos/eval/0_eval_0_3881364072b3e77caced.mp4 +2025-01-03 20:40:19,041 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/pc21xf38-media/videos/eval/0_eval_0_3881364072b3e77caced.mp4 +2025-01-03 20:40:19,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:19,666 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:40:20,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:20,750 DEBUG SystemMonitor:1968165 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 20:40:20,750 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:40:21,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:40:21,088 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:40:21,129 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:22,097 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:23,097 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:23,667 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:40:24,097 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:24,285 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:25,097 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:26,097 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:27,097 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:27,668 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:40:28,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:29,098 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:30,092 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:30,098 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:31,098 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:31,670 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:40:31,670 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:40:31,912 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/ksqkpkmt-events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:40:32,098 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:33,098 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:33,670 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:40:34,098 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:35,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:35,671 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:40:35,706 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:36,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:40:36,088 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:40:36,129 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:37,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:38,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:39,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:40,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:41,092 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:41,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:42,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:43,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:44,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:45,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:46,093 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:46,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:47,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:48,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:49,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:50,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:50,751 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:40:51,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:40:51,088 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:40:51,129 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:51,157 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:52,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:53,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:54,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:55,101 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:56,101 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:57,093 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:57,101 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:58,101 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:59,101 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:00,101 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:01,102 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:02,093 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:02,101 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:03,102 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:04,102 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:05,106 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:06,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:41:06,088 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:41:06,129 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:07,105 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:07,172 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:08,106 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:09,106 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:10,106 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:10,742 DEBUG SenderThread:1968165 [sender.py:send():379] send: files +2025-01-03 20:41:10,742 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_2f514b24221df96e8edb.mp4 with policy now +2025-01-03 20:41:10,745 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:41:10,745 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:41:10,746 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:41:10,766 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:41:11,007 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/uhq6eek3-media/videos/eval/1_eval_1_2f514b24221df96e8edb.mp4 +2025-01-03 20:41:11,108 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:11,682 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:41:11,683 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:41:11,683 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/1_eval_1_2f514b24221df96e8edb.mp4 +2025-01-03 20:41:11,683 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval +2025-01-03 20:41:11,960 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/y4xi87kj-media/videos/eval/1_eval_1_2f514b24221df96e8edb.mp4 +2025-01-03 20:41:12,108 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:12,683 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:41:13,093 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:13,108 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:14,116 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:15,115 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:15,684 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:41:16,118 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:17,122 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:18,110 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:18,309 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:19,110 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:19,685 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:41:20,110 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:20,751 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:41:21,090 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:41:21,090 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:41:21,133 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:22,111 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:23,111 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:23,563 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:41:23,564 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:41:23,564 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:23,565 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:41:23,687 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:41:23,687 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:41:24,111 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:25,111 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:25,687 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:41:26,112 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:27,111 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:27,688 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:41:28,112 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:28,951 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:29,112 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:29,689 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:41:30,112 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:31,112 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:32,112 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:33,112 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:34,095 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:34,113 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:35,113 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:36,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:41:36,089 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:41:36,133 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:37,113 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:38,113 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:39,114 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:39,572 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:40,114 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:41,114 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:42,114 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:43,114 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:43,693 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:41:44,114 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:44,797 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:45,115 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:46,115 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:47,115 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:48,115 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:49,115 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:50,097 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:50,115 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:50,752 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:41:51,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:41:51,089 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:41:51,133 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:52,116 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:53,116 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:54,116 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:55,116 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:55,180 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:56,116 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:57,116 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:58,117 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:59,117 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:00,117 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:01,095 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:01,117 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:02,120 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:03,066 DEBUG SenderThread:1968165 [sender.py:send():379] send: files +2025-01-03 20:42:03,066 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_ebd3ab9d4a47a5995972.mp4 with policy now +2025-01-03 20:42:03,074 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:42:03,074 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:42:03,074 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:42:03,086 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:42:03,122 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:03,360 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/yaogazxk-media/videos/eval/2_eval_2_ebd3ab9d4a47a5995972.mp4 +2025-01-03 20:42:03,700 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:42:03,700 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:42:03,700 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/2_eval_2_ebd3ab9d4a47a5995972.mp4 +2025-01-03 20:42:03,700 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval +2025-01-03 20:42:03,956 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/uccf9fdg-media/videos/eval/2_eval_2_ebd3ab9d4a47a5995972.mp4 +2025-01-03 20:42:04,119 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:05,121 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:05,700 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:42:06,091 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:42:06,091 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:42:06,137 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:06,183 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:07,122 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:08,125 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:09,121 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:09,702 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:42:10,121 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:11,123 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:11,702 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:42:12,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:12,122 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:13,123 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:14,125 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:15,126 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:16,130 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:17,136 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:18,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:18,132 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:19,134 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:20,133 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:20,752 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:42:21,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:42:21,094 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:42:21,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:22,137 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:23,136 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:23,240 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:24,140 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:25,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:26,138 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:27,145 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:28,143 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:29,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:29,138 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:30,140 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:31,139 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:31,425 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:42:31,425 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:42:31,426 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:42:31,709 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:42:31,709 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:42:31,946 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/na16n5k2-events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:42:32,139 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:33,139 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:33,709 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:42:34,145 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:34,405 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:35,181 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:36,092 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:42:36,092 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:42:36,139 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:37,140 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:37,711 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:42:38,140 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:39,140 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:40,140 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:40,197 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:41,140 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:42,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:43,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:44,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:45,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:46,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:46,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:47,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:48,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:49,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:50,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:50,753 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:42:51,092 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:42:51,092 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:42:51,142 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:51,155 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:52,142 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:53,142 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:54,142 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:55,142 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:56,142 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:57,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:57,142 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:58,143 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:59,143 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:00,145 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:01,147 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:02,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:02,147 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:03,147 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:04,147 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:05,147 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:05,720 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:06,092 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:43:06,092 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:43:06,147 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:07,147 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:07,248 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:08,147 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:09,148 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:10,148 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:11,151 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:11,559 DEBUG SenderThread:1968165 [sender.py:send():379] send: files +2025-01-03 20:43:11,559 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_caa22dfc15ca57c8e31d.mp4 with policy now +2025-01-03 20:43:11,562 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:43:11,563 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:43:11,563 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:43:11,563 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:43:11,722 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:43:11,722 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:11,723 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/3_eval_3_caa22dfc15ca57c8e31d.mp4 +2025-01-03 20:43:11,723 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval +2025-01-03 20:43:11,822 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/zb700243-media/videos/eval/3_eval_3_caa22dfc15ca57c8e31d.mp4 +2025-01-03 20:43:12,068 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/551o9nf6-media/videos/eval/3_eval_3_caa22dfc15ca57c8e31d.mp4 +2025-01-03 20:43:12,149 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:12,605 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:13,155 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:13,723 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:14,158 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:15,156 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:16,155 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:17,160 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:17,724 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:18,105 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:18,165 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:19,158 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:19,725 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:20,158 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:20,753 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:43:21,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:43:21,096 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:43:21,164 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:22,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:22,726 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:43:23,157 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:23,489 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:43:23,490 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:43:23,490 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:23,490 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:43:23,726 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:43:23,727 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:24,160 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:25,163 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:25,727 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:26,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:27,160 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:27,728 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:28,160 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:28,750 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:29,160 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:30,161 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:31,161 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:32,161 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:33,161 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:34,102 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:34,161 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:35,161 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:36,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:43:36,094 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:43:36,161 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:37,161 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:38,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:39,158 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:39,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:40,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:41,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:42,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:43,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:44,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:45,102 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:45,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:46,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:47,163 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:48,163 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:49,163 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:49,735 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:50,163 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:50,754 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:43:50,754 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:51,093 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:43:51,094 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:43:51,163 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:51,736 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:52,163 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:53,163 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:54,163 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:55,173 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:55,556 DEBUG SenderThread:1968165 [sender.py:send():379] send: files +2025-01-03 20:43:55,556 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ba85108b8a446815d41f.mp4 with policy now +2025-01-03 20:43:55,557 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:43:55,557 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:43:55,557 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:43:55,558 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:43:55,738 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:43:55,738 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:55,738 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/4_eval_4_ba85108b8a446815d41f.mp4 +2025-01-03 20:43:55,738 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval +2025-01-03 20:43:55,810 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/yg3gkggz-media/videos/eval/4_eval_4_ba85108b8a446815d41f.mp4 +2025-01-03 20:43:56,066 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/lptb1p13-media/videos/eval/4_eval_4_ba85108b8a446815d41f.mp4 +2025-01-03 20:43:56,110 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:56,175 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:57,177 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:57,739 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:58,176 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:59,180 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:59,740 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:44:00,183 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:01,111 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:44:01,184 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:02,183 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:03,204 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:03,742 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:44:04,191 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:05,192 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:06,101 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:44:06,101 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:44:06,188 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:06,258 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:44:07,191 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:08,194 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:09,193 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:10,193 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:11,204 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:12,121 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:44:12,216 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:13,200 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:14,217 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:15,219 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:16,228 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:17,229 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:18,118 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:44:18,235 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:19,224 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:20,229 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:20,754 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:44:21,097 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:44:21,097 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:44:21,226 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:22,242 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:23,238 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:23,244 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:44:24,231 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:25,238 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:26,239 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:27,243 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:28,241 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:29,116 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:44:29,242 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:29,568 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:44:29,568 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:29,569 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:29,744 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:44:29,744 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:44:29,751 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:44:29,751 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:44:29,883 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:44:30,035 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/7p4436g0-events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:44:30,239 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:30,267 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:44:30,309 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:44:30,329 INFO wandb-upload_0:1968165 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp3vfa9y6h +2025-01-03 20:44:30,749 INFO SenderThread:1968165 [sender.py:send_request_log_artifact():1525] logged artifact run-ts8zycr2-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg2MzYyOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3MzIwOA==', 'latestArtifact': None}} +2025-01-03 20:44:30,749 DEBUG SenderThread:1968165 [sender.py:send():379] send: files +2025-01-03 20:44:30,749 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json with policy now +2025-01-03 20:44:30,750 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:44:30,750 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,750 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,751 DEBUG SenderThread:1968165 [sender.py:send():379] send: telemetry +2025-01-03 20:44:30,751 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,751 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,751 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,753 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,753 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,754 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,754 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,754 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,755 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,755 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,755 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,755 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,756 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,756 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:44:30,756 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:44:30,756 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,756 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/table/eval/total_success_rate_ci_6_36b93f6d956bc7afa596.table.json +2025-01-03 20:44:30,757 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,757 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-03 20:44:30,757 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media +2025-01-03 20:44:30,757 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,758 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/table/eval +2025-01-03 20:44:30,758 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,758 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/table +2025-01-03 20:44:30,758 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,758 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,759 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,759 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,759 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,761 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:44:30,944 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/uawranxm-media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-03 20:44:30,996 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/8st80u84-media/table/eval/total_success_rate_ci_6_36b93f6d956bc7afa596.table.json +2025-01-03 20:44:31,102 INFO wandb-upload_2:1968165 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgxcxdh74 +2025-01-03 20:44:31,192 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/n8go47me-media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-03 20:44:31,239 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:31,469 INFO SenderThread:1968165 [sender.py:send_request_log_artifact():1525] logged artifact run-ts8zycr2-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg2MzY0MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3MzIxMA==', 'latestArtifact': None}} +2025-01-03 20:44:31,469 DEBUG SenderThread:1968165 [sender.py:send():379] send: files +2025-01-03 20:44:31,469 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_36b93f6d956bc7afa596.table.json with policy now +2025-01-03 20:44:31,469 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:44:31,469 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:31,470 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:31,710 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/he132pr8-media/table/eval/total_success_rate_ci_6_36b93f6d956bc7afa596.table.json +2025-01-03 20:44:31,756 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:44:31,756 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:44:32,239 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:33,240 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:34,240 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:34,470 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:44:35,240 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:35,613 DEBUG SenderThread:1968165 [sender.py:send():379] send: exit +2025-01-03 20:44:35,613 INFO SenderThread:1968165 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 20:44:35,614 INFO SenderThread:1968165 [sender.py:send_exit():588] handling runtime: 314 +2025-01-03 20:44:35,614 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:35,614 INFO SenderThread:1968165 [sender.py:send_exit():594] send defer +2025-01-03 20:44:35,614 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:35,614 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 20:44:35,614 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:35,614 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 20:44:35,614 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 1 +2025-01-03 20:44:35,615 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:35,615 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 20:44:35,615 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:35,615 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 20:44:35,615 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 2 +2025-01-03 20:44:35,615 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:35,615 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 20:44:35,615 INFO HandlerThread:1968165 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 20:44:35,615 DEBUG SystemMonitor:1968165 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 20:44:35,615 DEBUG SystemMonitor:1968165 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 20:44:35,616 INFO HandlerThread:1968165 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 20:44:35,616 INFO HandlerThread:1968165 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 20:44:35,668 INFO HandlerThread:1968165 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 20:44:35,669 INFO HandlerThread:1968165 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 20:44:35,669 INFO HandlerThread:1968165 [interfaces.py:finish():200] Joined network monitor +2025-01-03 20:44:35,670 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:35,670 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 20:44:35,670 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 3 +2025-01-03 20:44:35,670 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:44:35,670 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:35,670 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 20:44:35,671 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:35,671 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 20:44:35,671 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 4 +2025-01-03 20:44:35,671 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:35,671 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 20:44:35,758 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:44:37,759 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:44:42,573 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:44:42,573 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:44:42,573 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 20:44:42,574 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:42,574 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 20:44:42,575 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 5 +2025-01-03 20:44:42,575 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:44:42,575 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:44:42,575 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:42,575 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:42,575 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 20:44:42,576 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:42,576 DEBUG SenderThread:1968165 [sender.py:send():379] send: summary +2025-01-03 20:44:42,580 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:42,580 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:42,580 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 20:44:42,580 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 6 +2025-01-03 20:44:42,580 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:42,580 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 20:44:42,580 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:42,580 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 20:44:42,585 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:44:42,615 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:44:42,698 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 7 +2025-01-03 20:44:42,699 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:44:42,699 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:42,699 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 20:44:42,699 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:42,699 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 20:44:42,761 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/config.yaml +2025-01-03 20:44:42,761 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:44:43,615 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:44:44,870 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 8 +2025-01-03 20:44:44,870 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:44:44,870 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:44,870 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 20:44:44,870 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:44,870 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 20:44:44,870 INFO SenderThread:1968165 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 20:44:44,873 INFO SenderThread:1968165 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 20:44:44,874 INFO SenderThread:1968165 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 20:44:44,887 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 9 +2025-01-03 20:44:44,887 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:44,887 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 20:44:44,887 DEBUG SenderThread:1968165 [sender.py:send():379] send: artifact +2025-01-03 20:44:45,616 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:44:45,762 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:44:46,715 INFO SenderThread:1968165 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzg1NzYxOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzg1NzYxOA==', 'versionIndex': 407}}} +2025-01-03 20:44:46,715 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:46,716 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 20:44:46,716 INFO SenderThread:1968165 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 20:44:46,763 INFO SenderThread:1968165 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files +2025-01-03 20:44:46,763 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/config.yaml config.yaml +2025-01-03 20:44:46,763 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json wandb-summary.json +2025-01-03 20:44:46,763 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:44:46,763 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/requirements.txt requirements.txt +2025-01-03 20:44:46,763 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-metadata.json wandb-metadata.json +2025-01-03 20:44:46,764 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log output.log +2025-01-03 20:44:46,764 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/diff.patch diff.patch +2025-01-03 20:44:46,766 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 20:44:46,767 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/1_eval_1_2f514b24221df96e8edb.mp4 media/videos/eval/1_eval_1_2f514b24221df96e8edb.mp4 +2025-01-03 20:44:46,767 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/4_eval_4_ba85108b8a446815d41f.mp4 media/videos/eval/4_eval_4_ba85108b8a446815d41f.mp4 +2025-01-03 20:44:46,767 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/3_eval_3_caa22dfc15ca57c8e31d.mp4 media/videos/eval/3_eval_3_caa22dfc15ca57c8e31d.mp4 +2025-01-03 20:44:46,768 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/0_eval_0_3881364072b3e77caced.mp4 media/videos/eval/0_eval_0_3881364072b3e77caced.mp4 +2025-01-03 20:44:46,768 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/2_eval_2_ebd3ab9d4a47a5995972.mp4 media/videos/eval/2_eval_2_ebd3ab9d4a47a5995972.mp4 +2025-01-03 20:44:46,770 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/table/eval/total_success_rate_ci_6_36b93f6d956bc7afa596.table.json media/table/eval/total_success_rate_ci_6_36b93f6d956bc7afa596.table.json +2025-01-03 20:44:46,770 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-03 20:44:46,771 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 10 +2025-01-03 20:44:46,771 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:44:46,772 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:46,772 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 20:44:46,772 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:46,772 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 20:44:46,772 INFO SenderThread:1968165 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:44:46,969 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/config.yaml +2025-01-03 20:44:46,982 INFO wandb-upload_2:1968165 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:44:47,004 INFO wandb-upload_4:1968165 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:44:47,018 INFO wandb-upload_3:1968165 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/requirements.txt +2025-01-03 20:44:47,055 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/we44e1wu-events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:44:47,255 INFO Thread-11 (_thread_body):1968165 [sender.py:transition_state():614] send defer: 11 +2025-01-03 20:44:47,255 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:47,255 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 20:44:47,255 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:47,255 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 20:44:47,255 INFO SenderThread:1968165 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 20:44:47,257 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 12 +2025-01-03 20:44:47,257 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:47,257 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 20:44:47,257 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:47,257 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 20:44:47,257 INFO SenderThread:1968165 [file_stream.py:finish():601] file stream finish called +2025-01-03 20:44:47,319 INFO SenderThread:1968165 [file_stream.py:finish():605] file stream finish is done +2025-01-03 20:44:47,319 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 13 +2025-01-03 20:44:47,319 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:47,319 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 20:44:47,319 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:47,319 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 20:44:47,319 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 14 +2025-01-03 20:44:47,319 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:47,320 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 20:44:47,320 DEBUG SenderThread:1968165 [sender.py:send():379] send: final +2025-01-03 20:44:47,320 DEBUG SenderThread:1968165 [sender.py:send():379] send: footer +2025-01-03 20:44:47,320 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:47,320 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 20:44:47,320 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:44:47,320 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:44:47,321 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 20:44:47,321 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:44:47,321 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: server_info +2025-01-03 20:44:47,322 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 20:44:47,323 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 20:44:47,324 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:47,384 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:44:47,384 INFO MainThread:1968165 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 20:44:47,385 INFO MainThread:1968165 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 20:44:47,385 INFO MainThread:1968165 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 20:44:47,385 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 20:44:47,385 INFO HandlerThread:1968165 [handler.py:finish():882] shutting down handler +2025-01-03 20:44:48,321 INFO WriterThread:1968165 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/run-ts8zycr2.wandb +2025-01-03 20:44:48,384 INFO SenderThread:1968165 [sender.py:finish():1615] shutting down sender +2025-01-03 20:44:48,384 INFO SenderThread:1968165 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:44:48,385 INFO SenderThread:1968165 [file_pusher.py:join():175] waiting for file pusher diff --git a/ts8zycr2/wandb/debug.log b/ts8zycr2/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..818e4233caa313b18f478abdd4cec9c512dc5d96 --- /dev/null +++ b/ts8zycr2/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_setup.py:_flush():76] Configure stats pid to 1967991 +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/settings +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/logs/debug.log +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/logs/debug-internal.log +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_init.py:init():569] calling init triggers +2025-01-03 20:39:20,341 INFO MainThread:1967991 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2'} +2025-01-03 20:39:20,341 INFO MainThread:1967991 [wandb_init.py:init():619] starting backend +2025-01-03 20:39:20,341 INFO MainThread:1967991 [wandb_init.py:init():623] setting up manager +2025-01-03 20:39:20,344 INFO MainThread:1967991 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 20:39:20,345 INFO MainThread:1967991 [wandb_init.py:init():631] backend started and connected +2025-01-03 20:39:20,359 INFO MainThread:1967991 [wandb_init.py:init():720] updated telemetry +2025-01-03 20:39:20,365 INFO MainThread:1967991 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 20:39:20,666 INFO MainThread:1967991 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 20:39:20,737 INFO MainThread:1967991 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 20:39:20,737 INFO MainThread:1967991 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 20:39:21,087 INFO MainThread:1967991 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 20:39:21,087 INFO MainThread:1967991 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 20:39:21,087 INFO MainThread:1967991 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 20:39:21,088 INFO MainThread:1967991 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 20:39:21,089 INFO MainThread:1967991 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 20:39:21,089 INFO MainThread:1967991 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/tb, True +2025-01-03 20:39:25,128 INFO MainThread:1967991 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-03 20:44:48,386 WARNING MsgRouterThr:1967991 [router.py:message_loop():77] message_loop has been closed diff --git a/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/code/FISH/eval_robot.py b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/config.yaml b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de26836b19080187689aa289b09ef0d8a056a8ff --- /dev/null +++ b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1926_1' +wandb_notes: + desc: null + value: 1926_1_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735954760 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/diff.patch b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..643d2502c39fb837ef88b20f9de9cfed4f6af9cb --- /dev/null +++ b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/diff.patch @@ -0,0 +1,55 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..5ded7c9 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,17 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_2' # seed 2 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++wandb_run_id: '1926_1' # seed 1 ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json new file mode 100644 index 0000000000000000000000000000000000000000..a2db2594aac2dfee94d7430f5fc4332216ce7fe1 --- /dev/null +++ b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.8, 0.37485200169205224, 0.9957208297155283, 0.3695220351480134, 0.9926368444172731, 0.4414144158480513, 0.954582236616294, 0.4180309623643781, 0.934476241382488]]} \ No newline at end of file diff --git a/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/table/eval/total_success_rate_ci_6_36b93f6d956bc7afa596.table.json b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/table/eval/total_success_rate_ci_6_36b93f6d956bc7afa596.table.json new file mode 100644 index 0000000000000000000000000000000000000000..cebe5f4f52d7d699a0f350b1a452f615c752bcb1 --- /dev/null +++ b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/table/eval/total_success_rate_ci_6_36b93f6d956bc7afa596.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[9, 10, 0.9, 0.626920700047858, 0.9961568004390701, 0.5751074254362571, 0.9894275664304348, 0.6131431907189104, 0.990867448872502, 0.7759385108395876, 0.9852803832812113]]} \ No newline at end of file diff --git a/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/0_eval_0_3881364072b3e77caced.mp4 b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/0_eval_0_3881364072b3e77caced.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b1e22574edf752b88ab5cfe0248de102d163d93f --- /dev/null +++ b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/0_eval_0_3881364072b3e77caced.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3881364072b3e77caced7bb7be9deee5531f0ea3c1eb402de1f4ffb612b0542e +size 1439292 diff --git a/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/1_eval_1_2f514b24221df96e8edb.mp4 b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/1_eval_1_2f514b24221df96e8edb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0ada39852e82c62e7e335828b5f9a29a09608129 --- /dev/null +++ b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/1_eval_1_2f514b24221df96e8edb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f514b24221df96e8edb88f9e6195cfee832325974feead82d73898d6c40986b +size 1456453 diff --git a/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/2_eval_2_ebd3ab9d4a47a5995972.mp4 b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/2_eval_2_ebd3ab9d4a47a5995972.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..37c6425cf5e44b061d272116c1fdc570b5d8d87b --- /dev/null +++ b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/2_eval_2_ebd3ab9d4a47a5995972.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebd3ab9d4a47a599597242acc20388d3e1e4d82913dbccdeab479f251cd0fba2 +size 1451500 diff --git a/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/3_eval_3_caa22dfc15ca57c8e31d.mp4 b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/3_eval_3_caa22dfc15ca57c8e31d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5729eec17b708af7947b14d2551817768af111d4 --- /dev/null +++ b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/3_eval_3_caa22dfc15ca57c8e31d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caa22dfc15ca57c8e31dfaac858c43b2cddf3383e4b320501edb19c04f674bf5 +size 1467645 diff --git a/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/4_eval_4_ba85108b8a446815d41f.mp4 b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/4_eval_4_ba85108b8a446815d41f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fe53b6f340fa58ba6f48ef5f2222de1431496bea --- /dev/null +++ b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/4_eval_4_ba85108b8a446815d41f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba85108b8a446815d41f910e718c6f3d1fd0df3e53ccb6ead00253d8c730b7d1 +size 1310145 diff --git a/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e1ee5bc30782c0c0f5472fe7e67a6bfc55ccbbdf --- /dev/null +++ b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log @@ -0,0 +1,276 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735954765.129225]: resetting environment +[INFO] [1735954765.133915]: cleared current plan +[INFO] [1735954765.134899]: moving to home +[INFO] [1735954766.137255]: reached home +[INFO] [1735954766.138523]: reset action history +[INFO] [1735954768.850528]: environment reset +Starting episode 0 +[INFO] [1735954768.854517]: resetting environment +[INFO] [1735954768.860554]: cleared current plan +[INFO] [1735954768.861362]: moving to home +[INFO] [1735954769.864126]: reached home +[INFO] [1735954769.865575]: reset action history +[INFO] [1735954772.575187]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735954779.113039]: resetting environment +[INFO] [1735954779.120261]: cleared current plan +[INFO] [1735954779.121540]: moving to home +[INFO] [1735954780.124227]: reached home +[INFO] [1735954780.125486]: reset action history +[INFO] [1735954782.834404]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/episode_rosbags/episode_0_2025-01-03-20-39-42.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1967991.log +started roslaunch server http://158.130.50.37:46715/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1968271] +process[print_text-2]: started with pid [1968295] +process[rosbag_record-3]: started with pid [1968296] +started bagging! +For topic gripper_width: timestamp difference is -140895874 for nearest: 1735954811161600321 - target: 1735954811302496195 at idx 249 +[INFO] [1735954817.551302]: Storing episode... +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[print_text-2] killing on exit +[INFO] [1735954818.304298]: Stored episode 1. +[INFO] [1735954818.304622]: Saving video... +[INFO] [1735954818.511067]: Video saved! +Starting episode 1 +[INFO] [1735954818.523002]: resetting environment +[INFO] [1735954818.529301]: cleared current plan +[INFO] [1735954818.529533]: moving to home +[INFO] [1735954822.237492]: reached home +[INFO] [1735954822.238280]: reset action history +[INFO] [1735954824.949900]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735954830.748176]: resetting environment +[INFO] [1735954830.752248]: cleared current plan +[INFO] [1735954830.752428]: moving to home +[INFO] [1735954831.753211]: reached home +[INFO] [1735954831.755760]: reset action history +[INFO] [1735954834.467381]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/episode_rosbags/episode_1_2025-01-03-20-40-34.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1967991.log +started roslaunch server http://158.130.50.37:33055/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1968420] +process[print_text-5]: started with pid [1968421] +process[rosbag_record-6]: started with pid [1968445] +started bagging! +[INFO] [1735954869.326432]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1735954870.122671]: Stored episode 2. +[INFO] [1735954870.123468]: Saving video... +[INFO] [1735954870.714068]: Video saved! +Starting episode 2 +[INFO] [1735954870.746144]: resetting environment +[INFO] [1735954870.771448]: cleared current plan +[INFO] [1735954870.772106]: moving to home +[WARN] [1735954870.667079]: Plan exhausted +[WARN] [1735954870.707338]: Plan exhausted +[WARN] [1735954870.750185]: Plan exhausted +[INFO] [1735954874.478223]: reached home +[INFO] [1735954874.482016]: reset action history +[INFO] [1735954877.261253]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735954882.984211]: resetting environment +[INFO] [1735954882.995533]: cleared current plan +[INFO] [1735954882.995762]: moving to home +[INFO] [1735954883.997410]: reached home +[INFO] [1735954883.997863]: reset action history +[INFO] [1735954886.708537]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/episode_rosbags/episode_2_2025-01-03-20-41-26.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1967991.log +started roslaunch server http://158.130.50.37:43241/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1968563] +process[print_text-8]: started with pid [1968564] +process[rosbag_record-9]: started with pid [1968588] +started bagging! +For topic gripper_width: timestamp difference is -108617326 for nearest: 1735954901561597565 - target: 1735954901670214891 at idx 197 +[INFO] [1735954921.561577]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit[pub_text-7] killing on exit +[INFO] [1735954922.359226]: Stored episode 3. +[INFO] [1735954922.367059]: Saving video... +[WARN] [1735954922.945184]: Plan exhausted +[WARN] [1735954922.986377]: Plan exhausted +[WARN] [1735954923.023371]: Plan exhausted +[WARN] [1735954923.064041]: Plan exhausted +[INFO] [1735954923.030185]: Video saved! +Starting episode 3 +[INFO] [1735954923.076568]: resetting environment +[INFO] [1735954923.096707]: cleared current plan +[INFO] [1735954923.099144]: moving to home +[INFO] [1735954926.917469]: reached home +[INFO] [1735954926.923901]: reset action history +[INFO] [1735954929.703531]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735954951.330489]: resetting environment +[INFO] [1735954951.345788]: cleared current plan +[INFO] [1735954951.353418]: moving to home +[INFO] [1735954952.360386]: reached home +[INFO] [1735954952.362547]: reset action history +[INFO] [1735954955.138556]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/episode_rosbags/episode_3_2025-01-03-20-42-35.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1967991.log +started roslaunch server http://158.130.50.37:39113/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1968732] +process[print_text-11]: started with pid [1968733] +process[rosbag_record-12]: started with pid [1968734] +started bagging! +For topic gripper_width: timestamp difference is 104195499 for nearest: 1735954984928303861 - target: 1735954984824108362 at idx 249 +[INFO] [1735954990.379314]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1735954991.155818]: Stored episode 4. +[INFO] [1735954991.159995]: Saving video... +[INFO] [1735954991.508287]: Video saved! +Starting episode 4 +[INFO] [1735954991.570903]: resetting environment +[INFO] [1735954991.603564]: cleared current plan +[INFO] [1735954991.604501]: moving to home +[INFO] [1735954995.407490]: reached home +[INFO] [1735954995.412191]: reset action history +[INFO] [1735954998.213485]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735955002.656245]: resetting environment +[INFO] [1735955002.671922]: cleared current plan +[INFO] [1735955002.672829]: moving to home +[INFO] [1735955003.675608]: reached home +[INFO] [1735955003.691040]: reset action history +[INFO] [1735955006.482048]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/episode_rosbags/episode_4_2025-01-03-20-43-26.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1967991.log +started roslaunch server http://158.130.50.37:46249/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1968850] +process[print_text-14]: started with pid [1968851] +process[rosbag_record-15]: started with pid [1968875] +started bagging! +For topic gripper_width: timestamp difference is 106129157 for nearest: 1735955028628271090 - target: 1735955028522141933 at idx 249 +For topic gripper_width: timestamp difference is 107144517 for nearest: 1735955029928266210 - target: 1735955029821121693 at idx 249 +preempting episode +[INFO] [1735955034.339619]: Storing episode... +[print_text-14] killing on exit +[pub_text-13] killing on exit +[rosbag_record-15] killing on exit +[WARN] [1735955035.464415]: Plan exhausted +[WARN] [1735955035.503294]: Plan exhausted +[WARN] [1735955035.543160]: Plan exhausted +[WARN] [1735955035.583957]: Plan exhausted +[INFO] [1735955035.156835]: Stored episode 5. +[INFO] [1735955035.165320]: Saving video... +[INFO] [1735955035.508251]: Video saved! +[INFO] [1735955035.557985]: resetting environment +[INFO] [1735955035.586050]: cleared current plan +[INFO] [1735955035.586794]: moving to home +[INFO] [1735955038.889570]: reached home +[INFO] [1735955038.900972]: reset action history +[INFO] [1735955041.813317]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/requirements.txt b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-metadata.json b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..bdef332bb9d37f14a96e55aa18e222f4f90346cb --- /dev/null +++ b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T01:39:20.760201", + "startedAt": "2025-01-04T01:39:20.338752", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3943.6670000000004, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4500.25, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4511.948, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4505.875, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4501.688, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4502.934, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 746.461067199707 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..382ea33cb33d31cc2b7e78d7e290c63d9f71f881 --- /dev/null +++ b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "3881364072b3e77caced7bb7be9deee5531f0ea3c1eb402de1f4ffb612b0542e", "size": 1439292, "path": "media/videos/eval/0_eval_0_3881364072b3e77caced.mp4"}, "global_step": 637, "_timestamp": 1735955070.178771, "_runtime": 309.8325300216675, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "2f514b24221df96e8edb88f9e6195cfee832325974feead82d73898d6c40986b", "size": 1456453, "path": "media/videos/eval/1_eval_1_2f514b24221df96e8edb.mp4"}, "eval/num_success": 4.0, "episode": 10.0, "eval/success_rate": 0.800000011920929, "eval/success_rate_lb_0.01": 0.3748520016670227, "eval/success_rate_ub_0.01": 0.9957208037376404, "eval/success_rate_lb_0.025": 0.3695220351219177, "eval/success_rate_ub_0.025": 0.9926368594169617, "eval/success_rate_lb_0.05": 0.44141441583633423, "eval/success_rate_ub_0.05": 0.9545822143554688, "eval/success_rate_lb_0.1": 0.4180309772491455, "eval/success_rate_ub_0.1": 0.9344762563705444, "eval/2_eval": {"_type": "video-file", "sha256": "ebd3ab9d4a47a599597242acc20388d3e1e4d82913dbccdeab479f251cd0fba2", "size": 1451500, "path": "media/videos/eval/2_eval_2_ebd3ab9d4a47a5995972.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "caa22dfc15ca57c8e31dfaac858c43b2cddf3383e4b320501edb19c04f674bf5", "size": 1467645, "path": "media/videos/eval/3_eval_3_caa22dfc15ca57c8e31d.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "ba85108b8a446815d41f910e718c6f3d1fd0df3e53ccb6ead00253d8c730b7d1", "size": 1310145, "path": "media/videos/eval/4_eval_4_ba85108b8a446815d41f.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "8490adedbf62c026e30853c69820c3dd6a1c7da4ee481eab9e8dcf125e285d47", "size": 397, "artifact_path": "wandb-client-artifact://38dmbqvdem6ggxpt12tb8bask4zbrxi0n19qsnwqb96lyjjdsygwq07mogi701mvttqio0rjn9ll02aormvhcxed5v6en28fbfdew7v0f6pw2df47gwyjfb06jniyt1l/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://j25r4uwe5iai184bh3qbrmsoxofw7ea3uraccu36254avk5eso2f111z4ju2vl5vbjudw3cz4wjh70zlssegnwkjs8yvczqzg0vgjprwfkok7blu3bxjozhrbo45albv:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 9, "total_num_episodes": 10, "total_success_rate": 0.9, "total_success_rate_lb_0.01": 0.626920700047858, "total_success_rate_ub_0.01": 0.9961568004390701, "total_success_rate_lb_0.025": 0.5751074254362571, "total_success_rate_ub_0.025": 0.9894275664304348, "total_success_rate_lb_0.05": 0.6131431907189104, "total_success_rate_ub_0.05": 0.990867448872502, "total_success_rate_lb_0.1": 0.7759385108395876, "total_success_rate_ub_0.1": 0.9852803832812113, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "36b93f6d956bc7afa596f5dc0499bab70df4713303e9434eb7e9bd716558bf86", "size": 502, "artifact_path": "wandb-client-artifact://iyt7u93kq4ca2bn6uoain5h6a61hg726qbjjmt18moui2xnk7i1vjn6gpz5150nigvu89pw7xk5a8d1n9xb5fndtm9t4tuh8qlsx28fks2apjv1eeg2uqxmrva1hlzpf/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://eod3hj09dkne8xiy9b26kyyl6vu81f6dxk2ykt6i694z76xni0zs20t6gbys4qd2c0i81lctizsi40drf2b2aienodhhqqwue9qyuerqh4ch1mkf4yj3eqgtsubl0zwt:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_36b93f6d956bc7afa596.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 314}, "eval/total_success_rate": 0.8999999761581421, "eval/total_success_rate_lb_0.01": 0.6269207000732422, "eval/total_success_rate_ub_0.01": 0.9961568117141724, "eval/total_success_rate_lb_0.025": 0.5751073956489563, "eval/total_success_rate_ub_0.025": 0.9894275665283203, "eval/total_success_rate_lb_0.05": 0.6131432056427002, "eval/total_success_rate_ub_0.05": 0.9908674359321594, "eval/total_success_rate_lb_0.1": 0.7759385108947754, "eval/total_success_rate_ub_0.1": 0.9852803945541382} \ No newline at end of file diff --git a/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/logs/debug-internal.log b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1e9aca7076fb72087626b4cd883253b62cf5ed2a --- /dev/null +++ b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/logs/debug-internal.log @@ -0,0 +1,831 @@ +2025-01-03 20:39:20,345 INFO StreamThr :1968165 [internal.py:wandb_internal():85] W&B internal server running at pid: 1968165, started at: 2025-01-03 20:39:20.345539 +2025-01-03 20:39:20,346 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status +2025-01-03 20:39:20,359 INFO WriterThread:1968165 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/run-ts8zycr2.wandb +2025-01-03 20:39:20,359 DEBUG SenderThread:1968165 [sender.py:send():379] send: header +2025-01-03 20:39:20,367 DEBUG SenderThread:1968165 [sender.py:send():379] send: run +2025-01-03 20:39:20,649 INFO SenderThread:1968165 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files +2025-01-03 20:39:20,649 INFO SenderThread:1968165 [sender.py:_start_run_threads():1188] run started: ts8zycr2 with start time 1735954760.346241 +2025-01-03 20:39:20,667 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 20:39:20,667 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: check_version +2025-01-03 20:39:20,742 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 20:39:20,746 DEBUG HandlerThread:1968165 [system_info.py:__init__():26] System info init +2025-01-03 20:39:20,747 DEBUG HandlerThread:1968165 [system_info.py:__init__():41] System info init done +2025-01-03 20:39:20,747 INFO HandlerThread:1968165 [system_monitor.py:start():194] Starting system monitor +2025-01-03 20:39:20,747 INFO SystemMonitor:1968165 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 20:39:20,747 INFO HandlerThread:1968165 [system_monitor.py:probe():214] Collecting system info +2025-01-03 20:39:20,747 INFO SystemMonitor:1968165 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 20:39:20,748 INFO SystemMonitor:1968165 [interfaces.py:start():188] Started disk monitoring +2025-01-03 20:39:20,749 INFO SystemMonitor:1968165 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 20:39:20,749 INFO SystemMonitor:1968165 [interfaces.py:start():188] Started memory monitoring +2025-01-03 20:39:20,750 INFO SystemMonitor:1968165 [interfaces.py:start():188] Started network monitoring +2025-01-03 20:39:20,760 DEBUG HandlerThread:1968165 [system_info.py:probe():152] Probing system +2025-01-03 20:39:20,761 DEBUG HandlerThread:1968165 [system_info.py:_probe_git():137] Probing git +2025-01-03 20:39:20,766 DEBUG HandlerThread:1968165 [system_info.py:_probe_git():145] Probing git done +2025-01-03 20:39:20,766 DEBUG HandlerThread:1968165 [system_info.py:probe():200] Probing system done +2025-01-03 20:39:20,766 DEBUG HandlerThread:1968165 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T01:39:20.760201', 'startedAt': '2025-01-04T01:39:20.338752', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3943.6670000000004, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4500.25, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4511.948, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4505.875, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4501.688, 'min': 800.0, 'max': 4700.0}, {'current': 4502.934, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 746.461067199707}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 20:39:20,766 INFO HandlerThread:1968165 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 20:39:20,766 INFO HandlerThread:1968165 [system_monitor.py:probe():227] Publishing system info +2025-01-03 20:39:20,766 DEBUG HandlerThread:1968165 [system_info.py:_save_code():44] Saving code +2025-01-03 20:39:20,768 DEBUG HandlerThread:1968165 [system_info.py:_save_code():67] Saving code done +2025-01-03 20:39:20,768 DEBUG HandlerThread:1968165 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 20:39:20,831 DEBUG HandlerThread:1968165 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 20:39:20,831 INFO HandlerThread:1968165 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 20:39:20,835 DEBUG SenderThread:1968165 [sender.py:send():379] send: files +2025-01-03 20:39:20,835 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 20:39:20,835 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 20:39:20,835 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 20:39:21,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 20:39:21,087 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: python_packages +2025-01-03 20:39:21,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:21,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:39:21,089 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/zdme6q0t-wandb-metadata.json +2025-01-03 20:39:21,089 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:39:21,098 INFO wandb-upload_2:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/gdd2ad18-diff.patch +2025-01-03 20:39:21,106 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/q7txf2dp-code/FISH/eval_robot.py +2025-01-03 20:39:21,129 INFO HandlerThread:1968165 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/tb" + save: true +} +control { + relay_id: "e1fa12757c664fe9b3563e6e097209d1" +} +_info { + stream_id: "ts8zycr2" +} + +2025-01-03 20:39:21,130 DEBUG HandlerThread:1968165 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 20:39:21,231 DEBUG SenderThread:1968165 [sender.py:send():379] send: telemetry +2025-01-03 20:39:21,231 DEBUG SenderThread:1968165 [sender.py:send():379] send: tbrecord +2025-01-03 20:39:21,232 DEBUG SenderThread:1968165 [sender.py:send():379] send: files +2025-01-03 20:39:21,232 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 with policy live +2025-01-03 20:39:21,416 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/av6ooeil-events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:39:21,651 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-metadata.json +2025-01-03 20:39:21,651 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/requirements.txt +2025-01-03 20:39:21,651 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/code/FISH/eval_robot.py +2025-01-03 20:39:21,651 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/diff.patch +2025-01-03 20:39:21,652 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:39:21,652 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:39:21,652 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/code +2025-01-03 20:39:21,652 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/code/FISH +2025-01-03 20:39:22,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:23,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:23,650 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:39:24,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:25,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:25,129 DEBUG SenderThread:1968165 [sender.py:send():379] send: config +2025-01-03 20:39:26,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:39:26,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:27,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:27,651 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:39:28,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:29,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:30,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:31,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:39:31,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:31,653 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:39:32,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:33,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:33,653 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:39:34,089 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:35,089 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:36,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:39:36,087 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:39:36,129 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:36,209 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:39:37,089 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:38,089 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:39,089 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:40,089 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:41,089 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:41,655 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:39:42,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:39:42,089 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:43,091 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:44,090 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:45,090 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:45,656 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:39:46,090 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:47,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:39:47,090 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:48,090 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:49,091 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:50,091 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:51,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:39:51,087 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:39:51,133 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:52,091 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:52,205 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:39:52,658 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/config.yaml +2025-01-03 20:39:53,093 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:54,093 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:55,093 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:56,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:57,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:58,090 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:39:58,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:39:59,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:00,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:01,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:02,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:03,090 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:03,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:04,095 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:05,095 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:06,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:40:06,087 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:40:06,129 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:07,095 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:08,095 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:08,188 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:09,095 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:10,095 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:11,095 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:12,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:13,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:13,425 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:13,664 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:40:14,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:15,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:16,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:17,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:18,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:18,511 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:18,522 DEBUG SenderThread:1968165 [sender.py:send():379] send: files +2025-01-03 20:40:18,522 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_3881364072b3e77caced.mp4 with policy now +2025-01-03 20:40:18,522 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:40:18,523 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:40:18,523 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:40:18,524 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:40:18,665 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:40:18,665 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/0_eval_0_3881364072b3e77caced.mp4 +2025-01-03 20:40:18,666 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval +2025-01-03 20:40:18,666 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos +2025-01-03 20:40:18,666 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media +2025-01-03 20:40:18,790 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/1qeg35c1-media/videos/eval/0_eval_0_3881364072b3e77caced.mp4 +2025-01-03 20:40:19,041 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/pc21xf38-media/videos/eval/0_eval_0_3881364072b3e77caced.mp4 +2025-01-03 20:40:19,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:19,666 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:40:20,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:20,750 DEBUG SystemMonitor:1968165 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 20:40:20,750 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:40:21,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:40:21,088 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:40:21,129 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:22,097 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:23,097 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:23,667 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:40:24,097 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:24,285 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:25,097 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:26,097 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:27,097 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:27,668 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:40:28,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:29,098 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:30,092 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:30,098 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:31,098 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:31,670 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:40:31,670 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:40:31,912 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/ksqkpkmt-events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:40:32,098 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:33,098 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:33,670 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:40:34,098 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:35,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:35,671 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:40:35,706 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:36,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:40:36,088 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:40:36,129 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:37,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:38,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:39,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:40,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:41,092 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:41,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:42,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:43,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:44,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:45,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:46,093 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:46,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:47,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:48,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:49,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:50,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:50,751 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:40:51,087 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:40:51,088 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:40:51,129 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:51,157 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:52,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:53,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:54,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:55,101 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:56,101 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:57,093 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:40:57,101 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:58,101 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:40:59,101 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:00,101 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:01,102 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:02,093 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:02,101 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:03,102 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:04,102 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:05,106 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:06,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:41:06,088 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:41:06,129 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:07,105 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:07,172 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:08,106 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:09,106 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:10,106 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:10,742 DEBUG SenderThread:1968165 [sender.py:send():379] send: files +2025-01-03 20:41:10,742 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_2f514b24221df96e8edb.mp4 with policy now +2025-01-03 20:41:10,745 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:41:10,745 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:41:10,746 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:41:10,766 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:41:11,007 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/uhq6eek3-media/videos/eval/1_eval_1_2f514b24221df96e8edb.mp4 +2025-01-03 20:41:11,108 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:11,682 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:41:11,683 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:41:11,683 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/1_eval_1_2f514b24221df96e8edb.mp4 +2025-01-03 20:41:11,683 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval +2025-01-03 20:41:11,960 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/y4xi87kj-media/videos/eval/1_eval_1_2f514b24221df96e8edb.mp4 +2025-01-03 20:41:12,108 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:12,683 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:41:13,093 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:13,108 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:14,116 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:15,115 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:15,684 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:41:16,118 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:17,122 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:18,110 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:18,309 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:19,110 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:19,685 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:41:20,110 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:20,751 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:41:21,090 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:41:21,090 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:41:21,133 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:22,111 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:23,111 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:23,563 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:41:23,564 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:41:23,564 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:23,565 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:41:23,687 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:41:23,687 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:41:24,111 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:25,111 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:25,687 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:41:26,112 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:27,111 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:27,688 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:41:28,112 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:28,951 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:29,112 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:29,689 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:41:30,112 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:31,112 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:32,112 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:33,112 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:34,095 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:34,113 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:35,113 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:36,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:41:36,089 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:41:36,133 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:37,113 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:38,113 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:39,114 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:39,572 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:40,114 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:41,114 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:42,114 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:43,114 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:43,693 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:41:44,114 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:44,797 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:45,115 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:46,115 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:47,115 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:48,115 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:49,115 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:50,097 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:50,115 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:50,752 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:41:51,088 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:41:51,089 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:41:51,133 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:52,116 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:53,116 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:54,116 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:55,116 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:55,180 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:41:56,116 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:57,116 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:58,117 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:41:59,117 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:00,117 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:01,095 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:01,117 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:02,120 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:03,066 DEBUG SenderThread:1968165 [sender.py:send():379] send: files +2025-01-03 20:42:03,066 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_ebd3ab9d4a47a5995972.mp4 with policy now +2025-01-03 20:42:03,074 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:42:03,074 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:42:03,074 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:42:03,086 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:42:03,122 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:03,360 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/yaogazxk-media/videos/eval/2_eval_2_ebd3ab9d4a47a5995972.mp4 +2025-01-03 20:42:03,700 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:42:03,700 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:42:03,700 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/2_eval_2_ebd3ab9d4a47a5995972.mp4 +2025-01-03 20:42:03,700 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval +2025-01-03 20:42:03,956 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/uccf9fdg-media/videos/eval/2_eval_2_ebd3ab9d4a47a5995972.mp4 +2025-01-03 20:42:04,119 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:05,121 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:05,700 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:42:06,091 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:42:06,091 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:42:06,137 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:06,183 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:07,122 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:08,125 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:09,121 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:09,702 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:42:10,121 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:11,123 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:11,702 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:42:12,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:12,122 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:13,123 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:14,125 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:15,126 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:16,130 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:17,136 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:18,100 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:18,132 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:19,134 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:20,133 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:20,752 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:42:21,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:42:21,094 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:42:21,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:22,137 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:23,136 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:23,240 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:24,140 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:25,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:26,138 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:27,145 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:28,143 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:29,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:29,138 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:30,140 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:31,139 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:31,425 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:42:31,425 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:42:31,426 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:42:31,709 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:42:31,709 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:42:31,946 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/na16n5k2-events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:42:32,139 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:33,139 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:33,709 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:42:34,145 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:34,405 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:35,181 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:36,092 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:42:36,092 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:42:36,139 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:37,140 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:37,711 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:42:38,140 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:39,140 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:40,140 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:40,197 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:41,140 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:42,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:43,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:44,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:45,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:46,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:46,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:47,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:48,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:49,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:50,141 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:50,753 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:42:51,092 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:42:51,092 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:42:51,142 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:51,155 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:52,142 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:53,142 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:54,142 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:55,142 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:56,142 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:57,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:42:57,142 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:58,143 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:42:59,143 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:00,145 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:01,147 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:02,099 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:02,147 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:03,147 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:04,147 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:05,147 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:05,720 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:06,092 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:43:06,092 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:43:06,147 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:07,147 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:07,248 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:08,147 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:09,148 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:10,148 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:11,151 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:11,559 DEBUG SenderThread:1968165 [sender.py:send():379] send: files +2025-01-03 20:43:11,559 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_caa22dfc15ca57c8e31d.mp4 with policy now +2025-01-03 20:43:11,562 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:43:11,563 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:43:11,563 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:43:11,563 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:43:11,722 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:43:11,722 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:11,723 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/3_eval_3_caa22dfc15ca57c8e31d.mp4 +2025-01-03 20:43:11,723 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval +2025-01-03 20:43:11,822 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/zb700243-media/videos/eval/3_eval_3_caa22dfc15ca57c8e31d.mp4 +2025-01-03 20:43:12,068 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/551o9nf6-media/videos/eval/3_eval_3_caa22dfc15ca57c8e31d.mp4 +2025-01-03 20:43:12,149 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:12,605 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:13,155 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:13,723 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:14,158 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:15,156 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:16,155 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:17,160 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:17,724 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:18,105 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:18,165 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:19,158 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:19,725 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:20,158 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:20,753 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:43:21,096 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:43:21,096 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:43:21,164 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:22,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:22,726 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:43:23,157 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:23,489 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:43:23,490 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:43:23,490 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:23,490 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:43:23,726 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:43:23,727 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:24,160 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:25,163 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:25,727 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:26,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:27,160 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:27,728 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:28,160 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:28,750 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:29,160 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:30,161 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:31,161 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:32,161 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:33,161 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:34,102 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:34,161 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:35,161 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:36,094 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:43:36,094 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:43:36,161 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:37,161 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:38,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:39,158 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:39,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:40,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:41,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:42,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:43,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:44,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:45,102 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:45,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:46,162 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:47,163 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:48,163 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:49,163 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:49,735 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:50,163 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:50,754 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:43:50,754 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:51,093 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:43:51,094 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:43:51,163 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:51,736 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:52,163 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:53,163 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:54,163 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:55,173 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:55,556 DEBUG SenderThread:1968165 [sender.py:send():379] send: files +2025-01-03 20:43:55,556 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ba85108b8a446815d41f.mp4 with policy now +2025-01-03 20:43:55,557 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:43:55,557 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:43:55,557 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:43:55,558 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:43:55,738 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:43:55,738 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:55,738 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/4_eval_4_ba85108b8a446815d41f.mp4 +2025-01-03 20:43:55,738 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval +2025-01-03 20:43:55,810 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/yg3gkggz-media/videos/eval/4_eval_4_ba85108b8a446815d41f.mp4 +2025-01-03 20:43:56,066 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/lptb1p13-media/videos/eval/4_eval_4_ba85108b8a446815d41f.mp4 +2025-01-03 20:43:56,110 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:43:56,175 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:57,177 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:57,739 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:43:58,176 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:59,180 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:43:59,740 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:44:00,183 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:01,111 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:44:01,184 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:02,183 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:03,204 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:03,742 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:44:04,191 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:05,192 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:06,101 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:44:06,101 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:44:06,188 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:06,258 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:44:07,191 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:08,194 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:09,193 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:10,193 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:11,204 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:12,121 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:44:12,216 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:13,200 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:14,217 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:15,219 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:16,228 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:17,229 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:18,118 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:44:18,235 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:19,224 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:20,229 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:20,754 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:44:21,097 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:44:21,097 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:44:21,226 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:22,242 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:23,238 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:23,244 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:44:24,231 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:25,238 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:26,239 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:27,243 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:28,241 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:29,116 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:44:29,242 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:29,568 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:44:29,568 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:29,569 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:29,744 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:44:29,744 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:44:29,751 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:44:29,751 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:44:29,883 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:44:30,035 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/7p4436g0-events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:44:30,239 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:30,267 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:44:30,309 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:44:30,329 INFO wandb-upload_0:1968165 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp3vfa9y6h +2025-01-03 20:44:30,749 INFO SenderThread:1968165 [sender.py:send_request_log_artifact():1525] logged artifact run-ts8zycr2-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg2MzYyOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3MzIwOA==', 'latestArtifact': None}} +2025-01-03 20:44:30,749 DEBUG SenderThread:1968165 [sender.py:send():379] send: files +2025-01-03 20:44:30,749 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json with policy now +2025-01-03 20:44:30,750 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:44:30,750 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,750 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,751 DEBUG SenderThread:1968165 [sender.py:send():379] send: telemetry +2025-01-03 20:44:30,751 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,751 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,751 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,753 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,753 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,754 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,754 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,754 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,755 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,755 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,755 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,755 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,756 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,756 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:44:30,756 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:44:30,756 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,756 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/table/eval/total_success_rate_ci_6_36b93f6d956bc7afa596.table.json +2025-01-03 20:44:30,757 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,757 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-03 20:44:30,757 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media +2025-01-03 20:44:30,757 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,758 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/table/eval +2025-01-03 20:44:30,758 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,758 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/table +2025-01-03 20:44:30,758 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,758 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,759 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,759 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:30,759 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:30,761 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:44:30,944 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/uawranxm-media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-03 20:44:30,996 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/8st80u84-media/table/eval/total_success_rate_ci_6_36b93f6d956bc7afa596.table.json +2025-01-03 20:44:31,102 INFO wandb-upload_2:1968165 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgxcxdh74 +2025-01-03 20:44:31,192 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/n8go47me-media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-03 20:44:31,239 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:31,469 INFO SenderThread:1968165 [sender.py:send_request_log_artifact():1525] logged artifact run-ts8zycr2-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg2MzY0MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3MzIxMA==', 'latestArtifact': None}} +2025-01-03 20:44:31,469 DEBUG SenderThread:1968165 [sender.py:send():379] send: files +2025-01-03 20:44:31,469 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_36b93f6d956bc7afa596.table.json with policy now +2025-01-03 20:44:31,469 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:44:31,469 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:31,470 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:31,710 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/he132pr8-media/table/eval/total_success_rate_ci_6_36b93f6d956bc7afa596.table.json +2025-01-03 20:44:31,756 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:44:31,756 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:44:32,239 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:33,240 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:34,240 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:34,470 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:44:35,240 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:35,613 DEBUG SenderThread:1968165 [sender.py:send():379] send: exit +2025-01-03 20:44:35,613 INFO SenderThread:1968165 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 20:44:35,614 INFO SenderThread:1968165 [sender.py:send_exit():588] handling runtime: 314 +2025-01-03 20:44:35,614 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:35,614 INFO SenderThread:1968165 [sender.py:send_exit():594] send defer +2025-01-03 20:44:35,614 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:35,614 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 20:44:35,614 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:35,614 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 20:44:35,614 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 1 +2025-01-03 20:44:35,615 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:35,615 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 20:44:35,615 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:35,615 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 20:44:35,615 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 2 +2025-01-03 20:44:35,615 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:35,615 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 20:44:35,615 INFO HandlerThread:1968165 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 20:44:35,615 DEBUG SystemMonitor:1968165 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 20:44:35,615 DEBUG SystemMonitor:1968165 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 20:44:35,616 INFO HandlerThread:1968165 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 20:44:35,616 INFO HandlerThread:1968165 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 20:44:35,668 INFO HandlerThread:1968165 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 20:44:35,669 INFO HandlerThread:1968165 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 20:44:35,669 INFO HandlerThread:1968165 [interfaces.py:finish():200] Joined network monitor +2025-01-03 20:44:35,670 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:35,670 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 20:44:35,670 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 3 +2025-01-03 20:44:35,670 DEBUG SenderThread:1968165 [sender.py:send():379] send: stats +2025-01-03 20:44:35,670 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:35,670 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 20:44:35,671 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:35,671 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 20:44:35,671 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 4 +2025-01-03 20:44:35,671 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:35,671 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 20:44:35,758 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:44:37,759 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:44:42,573 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:44:42,573 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:44:42,573 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 20:44:42,574 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:42,574 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 20:44:42,575 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 5 +2025-01-03 20:44:42,575 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:44:42,575 DEBUG SenderThread:1968165 [sender.py:send():379] send: history +2025-01-03 20:44:42,575 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:44:42,575 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:42,575 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 20:44:42,576 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:42,576 DEBUG SenderThread:1968165 [sender.py:send():379] send: summary +2025-01-03 20:44:42,580 INFO SenderThread:1968165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:44:42,580 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:42,580 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 20:44:42,580 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 6 +2025-01-03 20:44:42,580 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:42,580 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 20:44:42,580 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:42,580 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 20:44:42,585 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:44:42,615 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:44:42,698 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 7 +2025-01-03 20:44:42,699 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:44:42,699 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:42,699 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 20:44:42,699 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:42,699 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 20:44:42,761 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/config.yaml +2025-01-03 20:44:42,761 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:44:43,615 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:44:44,870 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 8 +2025-01-03 20:44:44,870 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:44:44,870 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:44,870 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 20:44:44,870 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:44,870 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 20:44:44,870 INFO SenderThread:1968165 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 20:44:44,873 INFO SenderThread:1968165 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 20:44:44,874 INFO SenderThread:1968165 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 20:44:44,887 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 9 +2025-01-03 20:44:44,887 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:44,887 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 20:44:44,887 DEBUG SenderThread:1968165 [sender.py:send():379] send: artifact +2025-01-03 20:44:45,616 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:44:45,762 INFO Thread-12 :1968165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:44:46,715 INFO SenderThread:1968165 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzg1NzYxOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzg1NzYxOA==', 'versionIndex': 407}}} +2025-01-03 20:44:46,715 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:46,716 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 20:44:46,716 INFO SenderThread:1968165 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 20:44:46,763 INFO SenderThread:1968165 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files +2025-01-03 20:44:46,763 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/config.yaml config.yaml +2025-01-03 20:44:46,763 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json wandb-summary.json +2025-01-03 20:44:46,763 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:44:46,763 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/requirements.txt requirements.txt +2025-01-03 20:44:46,763 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-metadata.json wandb-metadata.json +2025-01-03 20:44:46,764 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log output.log +2025-01-03 20:44:46,764 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/diff.patch diff.patch +2025-01-03 20:44:46,766 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 20:44:46,767 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/1_eval_1_2f514b24221df96e8edb.mp4 media/videos/eval/1_eval_1_2f514b24221df96e8edb.mp4 +2025-01-03 20:44:46,767 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/4_eval_4_ba85108b8a446815d41f.mp4 media/videos/eval/4_eval_4_ba85108b8a446815d41f.mp4 +2025-01-03 20:44:46,767 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/3_eval_3_caa22dfc15ca57c8e31d.mp4 media/videos/eval/3_eval_3_caa22dfc15ca57c8e31d.mp4 +2025-01-03 20:44:46,768 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/0_eval_0_3881364072b3e77caced.mp4 media/videos/eval/0_eval_0_3881364072b3e77caced.mp4 +2025-01-03 20:44:46,768 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/videos/eval/2_eval_2_ebd3ab9d4a47a5995972.mp4 media/videos/eval/2_eval_2_ebd3ab9d4a47a5995972.mp4 +2025-01-03 20:44:46,770 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/table/eval/total_success_rate_ci_6_36b93f6d956bc7afa596.table.json media/table/eval/total_success_rate_ci_6_36b93f6d956bc7afa596.table.json +2025-01-03 20:44:46,770 INFO SenderThread:1968165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-03 20:44:46,771 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 10 +2025-01-03 20:44:46,771 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:44:46,772 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:46,772 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 20:44:46,772 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:46,772 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 20:44:46,772 INFO SenderThread:1968165 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:44:46,969 INFO wandb-upload_1:1968165 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/config.yaml +2025-01-03 20:44:46,982 INFO wandb-upload_2:1968165 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/wandb-summary.json +2025-01-03 20:44:47,004 INFO wandb-upload_4:1968165 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/output.log +2025-01-03 20:44:47,018 INFO wandb-upload_3:1968165 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/files/requirements.txt +2025-01-03 20:44:47,055 INFO wandb-upload_0:1968165 [upload_job.py:push():130] Uploaded file /tmp/tmp5eq_rvaowandb/we44e1wu-events.out.tfevents.1735954761.leonmkim-ROG-Strix-G15CS-G15CS.1967991.0 +2025-01-03 20:44:47,255 INFO Thread-11 (_thread_body):1968165 [sender.py:transition_state():614] send defer: 11 +2025-01-03 20:44:47,255 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:47,255 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 20:44:47,255 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:47,255 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 20:44:47,255 INFO SenderThread:1968165 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 20:44:47,257 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 12 +2025-01-03 20:44:47,257 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:47,257 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 20:44:47,257 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:47,257 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 20:44:47,257 INFO SenderThread:1968165 [file_stream.py:finish():601] file stream finish called +2025-01-03 20:44:47,319 INFO SenderThread:1968165 [file_stream.py:finish():605] file stream finish is done +2025-01-03 20:44:47,319 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 13 +2025-01-03 20:44:47,319 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:47,319 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 20:44:47,319 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:47,319 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 20:44:47,319 INFO SenderThread:1968165 [sender.py:transition_state():614] send defer: 14 +2025-01-03 20:44:47,319 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:44:47,320 INFO HandlerThread:1968165 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 20:44:47,320 DEBUG SenderThread:1968165 [sender.py:send():379] send: final +2025-01-03 20:44:47,320 DEBUG SenderThread:1968165 [sender.py:send():379] send: footer +2025-01-03 20:44:47,320 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: defer +2025-01-03 20:44:47,320 INFO SenderThread:1968165 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 20:44:47,320 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:44:47,320 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:44:47,321 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 20:44:47,321 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:44:47,321 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: server_info +2025-01-03 20:44:47,322 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 20:44:47,323 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 20:44:47,324 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:44:47,384 DEBUG SenderThread:1968165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:44:47,384 INFO MainThread:1968165 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 20:44:47,385 INFO MainThread:1968165 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 20:44:47,385 INFO MainThread:1968165 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 20:44:47,385 DEBUG HandlerThread:1968165 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 20:44:47,385 INFO HandlerThread:1968165 [handler.py:finish():882] shutting down handler +2025-01-03 20:44:48,321 INFO WriterThread:1968165 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/run-ts8zycr2.wandb +2025-01-03 20:44:48,384 INFO SenderThread:1968165 [sender.py:finish():1615] shutting down sender +2025-01-03 20:44:48,384 INFO SenderThread:1968165 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:44:48,385 INFO SenderThread:1968165 [file_pusher.py:join():175] waiting for file pusher diff --git a/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/logs/debug.log b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..818e4233caa313b18f478abdd4cec9c512dc5d96 --- /dev/null +++ b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_setup.py:_flush():76] Configure stats pid to 1967991 +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/settings +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/logs/debug.log +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/logs/debug-internal.log +2025-01-03 20:39:20,340 INFO MainThread:1967991 [wandb_init.py:init():569] calling init triggers +2025-01-03 20:39:20,341 INFO MainThread:1967991 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2'} +2025-01-03 20:39:20,341 INFO MainThread:1967991 [wandb_init.py:init():619] starting backend +2025-01-03 20:39:20,341 INFO MainThread:1967991 [wandb_init.py:init():623] setting up manager +2025-01-03 20:39:20,344 INFO MainThread:1967991 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 20:39:20,345 INFO MainThread:1967991 [wandb_init.py:init():631] backend started and connected +2025-01-03 20:39:20,359 INFO MainThread:1967991 [wandb_init.py:init():720] updated telemetry +2025-01-03 20:39:20,365 INFO MainThread:1967991 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 20:39:20,666 INFO MainThread:1967991 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 20:39:20,737 INFO MainThread:1967991 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 20:39:20,737 INFO MainThread:1967991 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 20:39:21,087 INFO MainThread:1967991 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 20:39:21,087 INFO MainThread:1967991 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 20:39:21,087 INFO MainThread:1967991 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 20:39:21,088 INFO MainThread:1967991 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 20:39:21,089 INFO MainThread:1967991 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 20:39:21,089 INFO MainThread:1967991 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/ts8zycr2/tb, True +2025-01-03 20:39:25,128 INFO MainThread:1967991 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-03 20:44:48,386 WARNING MsgRouterThr:1967991 [router.py:message_loop():77] message_loop has been closed diff --git a/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/run-ts8zycr2.wandb b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/run-ts8zycr2.wandb new file mode 100644 index 0000000000000000000000000000000000000000..146a47985a843a07875e2b3d83430bbdd4c7ca23 Binary files /dev/null and b/ts8zycr2/wandb/run-20250103_203920-ts8zycr2/run-ts8zycr2.wandb differ diff --git a/vgz4bwx7/.hydra/config.yaml b/vgz4bwx7/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/vgz4bwx7/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/vgz4bwx7/.hydra/hydra.yaml b/vgz4bwx7/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d69b6c8c1c481e1775e7ef02860b1880b85fc7f8 --- /dev/null +++ b/vgz4bwx7/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/vgz4bwx7/.hydra/overrides.yaml b/vgz4bwx7/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/vgz4bwx7/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/vgz4bwx7/episode_rosbags/aligned_depth_to_color_K.npy b/vgz4bwx7/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/vgz4bwx7/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/vgz4bwx7/episode_rosbags/cam_tf_world.npy b/vgz4bwx7/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/vgz4bwx7/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/vgz4bwx7/episode_rosbags/color_K.npy b/vgz4bwx7/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/vgz4bwx7/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/vgz4bwx7/episode_rosbags/depth_K.npy b/vgz4bwx7/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/vgz4bwx7/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/vgz4bwx7/episode_rosbags/episode_1_2025-01-03-21-21-55.bag b/vgz4bwx7/episode_rosbags/episode_1_2025-01-03-21-21-55.bag new file mode 100644 index 0000000000000000000000000000000000000000..b1147a7d2a1a0dc215f9bb1a3c37e2c3bc45224a --- /dev/null +++ b/vgz4bwx7/episode_rosbags/episode_1_2025-01-03-21-21-55.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2567eb68faac7bea20107c49f8b8d5daebb4b941e75acb4138c62cff24a0ea4d +size 918557254 diff --git a/vgz4bwx7/episode_rosbags/episode_2_2025-01-03-21-22-35.bag b/vgz4bwx7/episode_rosbags/episode_2_2025-01-03-21-22-35.bag new file mode 100644 index 0000000000000000000000000000000000000000..642362ff3d03b8b55813251eec29a49b21fb4f9d --- /dev/null +++ b/vgz4bwx7/episode_rosbags/episode_2_2025-01-03-21-22-35.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa948c761d37905db6e2090b8a062e6a89ad61916af9164ba4bcb153f3bc3703 +size 1447629639 diff --git a/vgz4bwx7/episode_rosbags/episode_3_2025-01-03-21-23-25.bag b/vgz4bwx7/episode_rosbags/episode_3_2025-01-03-21-23-25.bag new file mode 100644 index 0000000000000000000000000000000000000000..03faa4fd49eeb739a19b536d961a8e35552495ba --- /dev/null +++ b/vgz4bwx7/episode_rosbags/episode_3_2025-01-03-21-23-25.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa26527596066fa7832a1bcb2d99fad2e290207315bb287bf680f63d9096b78e +size 1332198922 diff --git a/vgz4bwx7/episode_rosbags/episode_4_2025-01-03-21-24-10.bag b/vgz4bwx7/episode_rosbags/episode_4_2025-01-03-21-24-10.bag new file mode 100644 index 0000000000000000000000000000000000000000..d9eed57bf7bb1af25b437d3f7c0b90fbe74a63a7 --- /dev/null +++ b/vgz4bwx7/episode_rosbags/episode_4_2025-01-03-21-24-10.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4262f95fa31610e1dd2c766e06e3012115469703a0e2a0039f3729f072a0fc9d +size 1152770818 diff --git a/vgz4bwx7/eval_robot.log b/vgz4bwx7/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..66ceb8211e976174f43d3ed924e64e0374d15a15 --- /dev/null +++ b/vgz4bwx7/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 21:20:38,241][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 21:20:38,244][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/vgz4bwx7/eval_video/0_eval.mp4 b/vgz4bwx7/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..85fa7267a8bd5845d43cca89c655f11624aef098 --- /dev/null +++ b/vgz4bwx7/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27c97977eb157644346f1978f0b0b20aee68b2d02d18588969187c8079b2ca5a +size 993819 diff --git a/vgz4bwx7/eval_video/1_eval.mp4 b/vgz4bwx7/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ad2d35ef170202fffee6d39640c6ae0de448902a --- /dev/null +++ b/vgz4bwx7/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac781709866b92b2b17b061e19df1d9075252fcc5d6b984dfc7757a9483d10e7 +size 601276 diff --git a/vgz4bwx7/eval_video/2_eval.mp4 b/vgz4bwx7/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df6b5c55eb41490d1b34d7c0426e33cbcc622639 --- /dev/null +++ b/vgz4bwx7/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dab3f95c8b865ba8aed6d5334650c6a14ef5d50ae1807606872a21792c0ae0b7 +size 882000 diff --git a/vgz4bwx7/eval_video/3_eval.mp4 b/vgz4bwx7/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7a0c7aaba24625679cedf4ace26dfd0e427752ec --- /dev/null +++ b/vgz4bwx7/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c216389b31307ee9d9e40ac7deb7ff69a0fbfc2ea7d9d271856e38bc3eb9bb54 +size 821461 diff --git a/vgz4bwx7/eval_video/4_eval.mp4 b/vgz4bwx7/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9e4ca776d1180a17d84af4aea1739f28c4e9d013 --- /dev/null +++ b/vgz4bwx7/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45ef389d8f994db06163105981839461bce48d04757a4e099c64220d28141532 +size 724143 diff --git a/vgz4bwx7/tb/events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 b/vgz4bwx7/tb/events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 new file mode 100644 index 0000000000000000000000000000000000000000..494d02bb364f79441b9dcc6e239212a36a523857 --- /dev/null +++ b/vgz4bwx7/tb/events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7255dcde1f66627d539149ee22651e81edd209d306cbbf32b7d2382f235af51 +size 6519 diff --git a/vgz4bwx7/wandb/debug-internal.log b/vgz4bwx7/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4d1660103d9c4e3c708245ab57de53078c8071bd --- /dev/null +++ b/vgz4bwx7/wandb/debug-internal.log @@ -0,0 +1,737 @@ +2025-01-03 21:20:43,964 INFO StreamThr :1976663 [internal.py:wandb_internal():85] W&B internal server running at pid: 1976663, started at: 2025-01-03 21:20:43.963679 +2025-01-03 21:20:43,966 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status +2025-01-03 21:20:43,977 INFO WriterThread:1976663 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/run-vgz4bwx7.wandb +2025-01-03 21:20:43,977 DEBUG SenderThread:1976663 [sender.py:send():379] send: header +2025-01-03 21:20:43,990 DEBUG SenderThread:1976663 [sender.py:send():379] send: run +2025-01-03 21:20:44,366 INFO SenderThread:1976663 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files +2025-01-03 21:20:44,366 INFO SenderThread:1976663 [sender.py:_start_run_threads():1188] run started: vgz4bwx7 with start time 1735957243.963966 +2025-01-03 21:20:44,383 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 21:20:44,383 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: check_version +2025-01-03 21:20:44,429 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 21:20:44,434 DEBUG HandlerThread:1976663 [system_info.py:__init__():26] System info init +2025-01-03 21:20:44,434 DEBUG HandlerThread:1976663 [system_info.py:__init__():41] System info init done +2025-01-03 21:20:44,434 INFO HandlerThread:1976663 [system_monitor.py:start():194] Starting system monitor +2025-01-03 21:20:44,434 INFO SystemMonitor:1976663 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 21:20:44,435 INFO HandlerThread:1976663 [system_monitor.py:probe():214] Collecting system info +2025-01-03 21:20:44,435 INFO SystemMonitor:1976663 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 21:20:44,436 INFO SystemMonitor:1976663 [interfaces.py:start():188] Started disk monitoring +2025-01-03 21:20:44,436 INFO SystemMonitor:1976663 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 21:20:44,437 INFO SystemMonitor:1976663 [interfaces.py:start():188] Started memory monitoring +2025-01-03 21:20:44,437 INFO SystemMonitor:1976663 [interfaces.py:start():188] Started network monitoring +2025-01-03 21:20:44,447 DEBUG HandlerThread:1976663 [system_info.py:probe():152] Probing system +2025-01-03 21:20:44,448 DEBUG HandlerThread:1976663 [system_info.py:_probe_git():137] Probing git +2025-01-03 21:20:44,453 DEBUG HandlerThread:1976663 [system_info.py:_probe_git():145] Probing git done +2025-01-03 21:20:44,453 DEBUG HandlerThread:1976663 [system_info.py:probe():200] Probing system done +2025-01-03 21:20:44,453 DEBUG HandlerThread:1976663 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T02:20:44.447582', 'startedAt': '2025-01-04T02:20:43.954878', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3936.0048750000005, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.968, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.707, 'min': 800.0, 'max': 4700.0}, {'current': 4497.939, 'min': 800.0, 'max': 4700.0}, {'current': 4496.57, 'min': 800.0, 'max': 4700.0}, {'current': 4499.962, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 812.3028678894043}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 21:20:44,453 INFO HandlerThread:1976663 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 21:20:44,453 INFO HandlerThread:1976663 [system_monitor.py:probe():227] Publishing system info +2025-01-03 21:20:44,453 DEBUG HandlerThread:1976663 [system_info.py:_save_code():44] Saving code +2025-01-03 21:20:44,455 DEBUG HandlerThread:1976663 [system_info.py:_save_code():67] Saving code done +2025-01-03 21:20:44,455 DEBUG HandlerThread:1976663 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 21:20:44,518 DEBUG HandlerThread:1976663 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 21:20:44,518 INFO HandlerThread:1976663 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 21:20:44,522 DEBUG SenderThread:1976663 [sender.py:send():379] send: files +2025-01-03 21:20:44,522 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 21:20:44,522 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 21:20:44,522 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 21:20:44,762 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 21:20:44,762 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: python_packages +2025-01-03 21:20:44,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:20:44,764 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:20:44,804 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/h94zdgtm-diff.patch +2025-01-03 21:20:44,805 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:44,805 INFO HandlerThread:1976663 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/tb" + save: true +} +control { + relay_id: "90088d12147245b2a5b6f7014b49871b" +} +_info { + stream_id: "vgz4bwx7" +} + +2025-01-03 21:20:44,805 DEBUG HandlerThread:1976663 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 21:20:44,814 INFO wandb-upload_1:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/un2iepj0-code/FISH/eval_robot.py +2025-01-03 21:20:44,818 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/qiejtn4c-wandb-metadata.json +2025-01-03 21:20:44,865 DEBUG SenderThread:1976663 [sender.py:send():379] send: telemetry +2025-01-03 21:20:44,874 DEBUG SenderThread:1976663 [sender.py:send():379] send: tbrecord +2025-01-03 21:20:44,875 DEBUG SenderThread:1976663 [sender.py:send():379] send: files +2025-01-03 21:20:44,875 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 with policy live +2025-01-03 21:20:45,058 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/mehq0b1b-events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:20:45,368 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/requirements.txt +2025-01-03 21:20:45,368 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:20:45,368 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:20:45,368 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-metadata.json +2025-01-03 21:20:45,368 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/diff.patch +2025-01-03 21:20:45,368 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/code/FISH/eval_robot.py +2025-01-03 21:20:45,369 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/code/FISH +2025-01-03 21:20:45,369 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/code +2025-01-03 21:20:45,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:46,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:47,367 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:20:47,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:48,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:49,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:20:49,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:50,584 DEBUG SenderThread:1976663 [sender.py:send():379] send: config +2025-01-03 21:20:50,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:51,368 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:20:51,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:52,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:53,369 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:20:53,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:54,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:20:54,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:55,369 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:20:55,765 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:56,765 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:57,370 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:20:57,765 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:58,765 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:59,371 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:20:59,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:20:59,763 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:20:59,805 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:59,847 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:00,765 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:01,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:02,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:03,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:04,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:05,372 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:21:05,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:05,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:06,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:07,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:08,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:09,373 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:21:09,767 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:10,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:10,767 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:11,767 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:12,767 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:13,767 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:14,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:21:14,763 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:21:14,805 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:15,768 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:15,828 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:16,375 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/config.yaml +2025-01-03 21:21:16,768 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:17,768 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:18,768 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:19,768 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:20,768 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:21,765 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:21,769 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:22,769 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:23,769 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:24,769 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:25,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:26,765 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:26,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:27,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:28,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:29,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:21:29,763 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:21:29,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:30,202 DEBUG SenderThread:1976663 [sender.py:send():379] send: files +2025-01-03 21:21:30,202 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_27c97977eb157644346f.mp4 with policy now +2025-01-03 21:21:30,202 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:21:30,203 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:21:30,203 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:21:30,204 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:21:30,379 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:21:30,379 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/0_eval_0_27c97977eb157644346f.mp4 +2025-01-03 21:21:30,379 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media +2025-01-03 21:21:30,379 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos +2025-01-03 21:21:30,379 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval +2025-01-03 21:21:30,424 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/65hkwpps-media/videos/eval/0_eval_0_27c97977eb157644346f.mp4 +2025-01-03 21:21:30,676 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/p1uuy1s5-media/videos/eval/0_eval_0_27c97977eb157644346f.mp4 +2025-01-03 21:21:30,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:31,379 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:21:31,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:32,213 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:32,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:33,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:34,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:35,381 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:21:35,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:36,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:37,673 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:37,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:38,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:39,382 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:21:39,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:40,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:41,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:42,765 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:42,772 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:43,772 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:44,437 DEBUG SystemMonitor:1976663 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 21:21:44,438 DEBUG SenderThread:1976663 [sender.py:send():379] send: stats +2025-01-03 21:21:44,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:21:44,764 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:21:44,805 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:45,772 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:46,772 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:47,772 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:47,919 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:48,773 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:49,773 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:50,773 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:51,386 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:21:51,618 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/tynca6yv-events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:21:51,773 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:52,773 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:53,385 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:53,387 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:21:53,773 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:54,773 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:55,773 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:56,774 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:57,388 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:21:57,774 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:58,774 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:59,305 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:59,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:21:59,764 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:21:59,805 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:00,774 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:01,774 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:02,774 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:03,774 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:04,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:04,774 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:05,775 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:06,775 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:07,775 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:08,778 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:08,921 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:22:08,921 DEBUG SenderThread:1976663 [sender.py:send():379] send: files +2025-01-03 21:22:08,921 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_ac781709866b92b2b17b.mp4 with policy now +2025-01-03 21:22:08,931 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:22:08,931 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:22:08,932 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:22:09,132 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/ag7wzq5w-media/videos/eval/1_eval_1_ac781709866b92b2b17b.mp4 +2025-01-03 21:22:09,392 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:22:09,392 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:22:09,392 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/1_eval_1_ac781709866b92b2b17b.mp4 +2025-01-03 21:22:09,392 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval +2025-01-03 21:22:09,647 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/hxupvmqv-media/videos/eval/1_eval_1_ac781709866b92b2b17b.mp4 +2025-01-03 21:22:09,777 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:09,780 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:10,777 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:11,393 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:22:11,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:12,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:13,779 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:14,439 DEBUG SenderThread:1976663 [sender.py:send():379] send: stats +2025-01-03 21:22:14,769 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:22:14,769 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:22:14,813 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:14,834 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:15,394 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:22:15,779 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:16,779 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:17,395 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:22:17,779 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:18,779 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:19,779 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:20,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:20,780 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:21,780 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:22,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:23,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:24,780 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:25,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:25,780 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:26,780 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:27,780 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:28,780 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:29,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:22:29,765 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:22:29,805 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:30,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:30,876 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:31,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:32,245 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:22:32,245 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:22:32,246 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:22:32,400 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:22:32,400 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:22:32,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:33,400 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:22:33,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:34,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:35,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:36,404 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:36,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:37,401 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:22:37,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:38,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:39,782 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:40,782 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:41,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:41,782 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:42,782 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:43,782 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:44,439 DEBUG SenderThread:1976663 [sender.py:send():379] send: stats +2025-01-03 21:22:44,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:22:44,765 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:22:44,805 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:45,784 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:46,785 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:46,839 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:47,784 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:48,784 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:49,785 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:50,784 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:51,785 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:52,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:52,785 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:53,785 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:54,785 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:55,795 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:56,215 DEBUG SenderThread:1976663 [sender.py:send():379] send: files +2025-01-03 21:22:56,215 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_dab3f95c8b865ba8aed6.mp4 with policy now +2025-01-03 21:22:56,218 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:22:56,218 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:22:56,218 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:22:56,219 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:22:56,408 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:22:56,408 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/2_eval_2_dab3f95c8b865ba8aed6.mp4 +2025-01-03 21:22:56,408 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval +2025-01-03 21:22:56,426 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/pixxa322-media/videos/eval/2_eval_2_dab3f95c8b865ba8aed6.mp4 +2025-01-03 21:22:56,708 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/j7tl2ri7-media/videos/eval/2_eval_2_dab3f95c8b865ba8aed6.mp4 +2025-01-03 21:22:56,789 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:57,408 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:22:57,793 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:58,231 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:58,791 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:59,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:22:59,766 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:22:59,809 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:00,792 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:01,409 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:01,790 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:02,881 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:03,879 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:03,941 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:04,875 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:05,411 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:05,878 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:06,878 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:07,892 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:08,880 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:09,775 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:09,881 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:10,881 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:11,886 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:12,885 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:13,888 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:14,439 DEBUG SenderThread:1976663 [sender.py:send():379] send: stats +2025-01-03 21:23:14,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:23:14,766 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:23:14,883 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:14,889 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:15,885 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:16,887 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:17,891 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:18,884 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:19,887 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:20,779 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:20,887 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:21,416 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:23:21,659 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/y6506mdo-events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:23:21,889 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:22,093 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:23:22,093 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:23:22,093 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:23:22,416 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:23:22,887 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:23,416 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:23,887 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:24,891 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:25,888 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:26,429 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:26,888 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:27,418 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:27,888 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:28,888 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:29,765 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:23:29,765 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:23:29,888 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:30,888 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:31,888 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:31,906 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:32,889 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:33,889 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:34,889 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:35,889 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:36,889 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:37,421 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:37,585 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:37,889 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:38,890 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:39,890 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:40,890 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:41,890 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:42,890 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:43,158 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:43,423 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:43,892 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:44,440 DEBUG SenderThread:1976663 [sender.py:send():379] send: stats +2025-01-03 21:23:44,739 DEBUG SenderThread:1976663 [sender.py:send():379] send: files +2025-01-03 21:23:44,739 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c216389b31307ee9d9e4.mp4 with policy now +2025-01-03 21:23:44,744 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:23:44,744 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:23:44,745 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:23:44,745 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:23:44,775 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:23:44,775 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:23:44,890 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:44,981 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/m8fqdeit-media/videos/eval/3_eval_3_c216389b31307ee9d9e4.mp4 +2025-01-03 21:23:45,424 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:45,424 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:23:45,424 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/3_eval_3_c216389b31307ee9d9e4.mp4 +2025-01-03 21:23:45,425 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval +2025-01-03 21:23:45,671 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/1bl9vy5b-media/videos/eval/3_eval_3_c216389b31307ee9d9e4.mp4 +2025-01-03 21:23:45,897 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:46,896 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:47,425 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:47,895 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:48,391 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:48,902 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:49,426 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:49,900 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:50,912 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:51,896 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:52,899 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:53,427 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:53,903 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:54,185 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:54,908 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:55,901 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:56,901 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:57,905 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:58,912 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:59,772 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:23:59,772 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:23:59,901 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:59,904 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:00,912 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:01,906 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:02,911 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:03,912 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:04,909 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:05,792 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:05,909 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:06,151 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:24:06,151 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:06,151 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:06,431 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:24:06,431 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:24:06,911 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:07,432 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:07,914 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:08,913 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:09,432 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:09,915 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:10,913 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:11,433 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:11,468 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:11,913 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:12,913 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:13,914 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:14,440 DEBUG SenderThread:1976663 [sender.py:send():379] send: stats +2025-01-03 21:24:14,769 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:24:14,769 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:24:14,913 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:15,915 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:16,857 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:16,915 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:17,915 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:18,915 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:19,916 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:20,916 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:21,916 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:22,789 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:22,916 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:23,916 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:24,916 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:25,917 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:26,942 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:27,025 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:24:27,026 DEBUG SenderThread:1976663 [sender.py:send():379] send: files +2025-01-03 21:24:27,026 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_45ef389d8f994db06163.mp4 with policy now +2025-01-03 21:24:27,026 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:24:27,026 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:27,027 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:27,233 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/gvz5rqpa-media/videos/eval/4_eval_4_45ef389d8f994db06163.mp4 +2025-01-03 21:24:27,439 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:27,439 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:24:27,439 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/4_eval_4_45ef389d8f994db06163.mp4 +2025-01-03 21:24:27,439 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval +2025-01-03 21:24:27,684 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/2abo3b9s-media/videos/eval/4_eval_4_45ef389d8f994db06163.mp4 +2025-01-03 21:24:27,945 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:28,028 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:28,944 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:29,439 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:29,779 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:24:29,779 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:24:29,944 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:30,952 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:31,440 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:31,943 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:32,945 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:33,441 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:33,514 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:33,946 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:34,959 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:35,441 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:35,945 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:36,951 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:37,953 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:38,797 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:38,951 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:39,956 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:40,961 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:41,957 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:42,963 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:43,960 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:44,496 DEBUG SenderThread:1976663 [sender.py:send():379] send: stats +2025-01-03 21:24:44,497 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:44,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:24:44,771 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:24:44,963 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:45,966 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:46,966 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:47,968 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:48,967 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:49,446 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:24:49,692 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/x7c1goys-events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:24:49,793 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:49,858 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 21:24:49,858 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 21:24:49,964 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:50,120 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:24:50,316 INFO wandb-upload_2:1976663 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpoe0_7d8x +2025-01-03 21:24:50,446 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:24:50,447 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-03 21:24:50,447 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media +2025-01-03 21:24:50,447 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table +2025-01-03 21:24:50,447 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table/eval +2025-01-03 21:24:50,491 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 21:24:50,503 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:24:50,729 INFO SenderThread:1976663 [sender.py:send_request_log_artifact():1525] logged artifact run-vgz4bwx7-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzkwOTY0NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3ODkzNg==', 'latestArtifact': None}} +2025-01-03 21:24:50,729 DEBUG SenderThread:1976663 [sender.py:send():379] send: files +2025-01-03 21:24:50,729 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json with policy now +2025-01-03 21:24:50,729 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:24:50,729 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,730 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,730 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:24:50,730 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,734 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,734 DEBUG SenderThread:1976663 [sender.py:send():379] send: telemetry +2025-01-03 21:24:50,734 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,734 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,734 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,735 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,735 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,735 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,735 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,736 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,736 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,736 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,736 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,736 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,737 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,737 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,737 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,737 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,738 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,738 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,738 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,738 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,739 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,739 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,739 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 21:24:50,912 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/9lxpfd20-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-03 21:24:50,962 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:51,144 INFO wandb-upload_1:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/s201i0pv-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-03 21:24:51,150 INFO wandb-upload_2:1976663 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4hy42nnt +2025-01-03 21:24:51,447 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:51,447 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:24:51,447 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table/eval/total_success_rate_ci_6_b5c7077c1f7a17f44586.table.json +2025-01-03 21:24:51,447 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table/eval +2025-01-03 21:24:51,593 INFO SenderThread:1976663 [sender.py:send_request_log_artifact():1525] logged artifact run-vgz4bwx7-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzkwOTY3NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3ODkzOQ==', 'latestArtifact': None}} +2025-01-03 21:24:51,593 DEBUG SenderThread:1976663 [sender.py:send():379] send: files +2025-01-03 21:24:51,593 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_b5c7077c1f7a17f44586.table.json with policy now +2025-01-03 21:24:51,594 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:24:51,594 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:51,594 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:51,810 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/jqv5cze4-media/table/eval/total_success_rate_ci_6_b5c7077c1f7a17f44586.table.json +2025-01-03 21:24:51,962 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:52,098 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/b3c8flsg-media/table/eval/total_success_rate_ci_6_b5c7077c1f7a17f44586.table.json +2025-01-03 21:24:52,447 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:24:52,813 DEBUG SenderThread:1976663 [sender.py:send():379] send: exit +2025-01-03 21:24:52,813 INFO SenderThread:1976663 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 21:24:52,814 INFO SenderThread:1976663 [sender.py:send_exit():588] handling runtime: 248 +2025-01-03 21:24:52,814 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:52,814 INFO SenderThread:1976663 [sender.py:send_exit():594] send defer +2025-01-03 21:24:52,814 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:24:52,814 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 21:24:52,814 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:24:52,814 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 21:24:52,815 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 1 +2025-01-03 21:24:52,815 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:24:52,815 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 21:24:52,815 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:24:52,815 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 21:24:52,815 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 2 +2025-01-03 21:24:52,815 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:24:52,815 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 21:24:52,815 INFO HandlerThread:1976663 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 21:24:52,815 DEBUG SystemMonitor:1976663 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 21:24:52,816 DEBUG SystemMonitor:1976663 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 21:24:52,817 INFO HandlerThread:1976663 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 21:24:52,817 INFO HandlerThread:1976663 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 21:24:52,868 INFO HandlerThread:1976663 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 21:24:52,868 INFO HandlerThread:1976663 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 21:24:52,869 INFO HandlerThread:1976663 [interfaces.py:finish():200] Joined network monitor +2025-01-03 21:24:52,869 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:24:52,869 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 21:24:52,869 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 3 +2025-01-03 21:24:52,869 DEBUG SenderThread:1976663 [sender.py:send():379] send: stats +2025-01-03 21:24:52,869 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:24:52,869 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 21:24:52,870 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:24:52,870 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 21:24:52,870 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 4 +2025-01-03 21:24:52,870 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:24:52,870 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 21:24:53,448 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:53,448 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:24:55,449 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:59,211 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:24:59,211 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:59,211 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 21:24:59,211 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:24:59,212 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 21:24:59,212 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 5 +2025-01-03 21:24:59,212 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:24:59,213 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:24:59,213 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 21:24:59,213 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:24:59,213 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:59,214 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:59,214 DEBUG SenderThread:1976663 [sender.py:send():379] send: summary +2025-01-03 21:24:59,214 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:59,214 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:24:59,214 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 21:24:59,214 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 6 +2025-01-03 21:24:59,215 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:24:59,215 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 21:24:59,215 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:24:59,215 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 21:24:59,218 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:59,358 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 7 +2025-01-03 21:24:59,358 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:24:59,358 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 21:24:59,358 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:24:59,358 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 21:24:59,450 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:24:59,450 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/config.yaml +2025-01-03 21:24:59,815 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:25:03,041 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 8 +2025-01-03 21:25:03,042 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:25:03,042 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:25:03,042 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 21:25:03,042 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:25:03,043 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 21:25:03,043 INFO SenderThread:1976663 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 21:25:03,048 INFO SenderThread:1976663 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 21:25:03,051 INFO SenderThread:1976663 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 21:25:03,062 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 9 +2025-01-03 21:25:03,062 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:25:03,062 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 21:25:03,062 DEBUG SenderThread:1976663 [sender.py:send():379] send: artifact +2025-01-03 21:25:03,265 INFO SenderThread:1976663 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzkwNDU3OA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzkwNDU3OA==', 'versionIndex': 410}}} +2025-01-03 21:25:03,265 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:25:03,265 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 21:25:03,266 INFO SenderThread:1976663 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 21:25:03,451 INFO SenderThread:1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:25:03,452 INFO SenderThread:1976663 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files +2025-01-03 21:25:03,452 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/config.yaml config.yaml +2025-01-03 21:25:03,452 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json wandb-summary.json +2025-01-03 21:25:03,452 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/requirements.txt requirements.txt +2025-01-03 21:25:03,452 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-metadata.json wandb-metadata.json +2025-01-03 21:25:03,452 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:25:03,452 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log output.log +2025-01-03 21:25:03,452 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/diff.patch diff.patch +2025-01-03 21:25:03,454 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 21:25:03,457 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/2_eval_2_dab3f95c8b865ba8aed6.mp4 media/videos/eval/2_eval_2_dab3f95c8b865ba8aed6.mp4 +2025-01-03 21:25:03,457 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/3_eval_3_c216389b31307ee9d9e4.mp4 media/videos/eval/3_eval_3_c216389b31307ee9d9e4.mp4 +2025-01-03 21:25:03,457 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/1_eval_1_ac781709866b92b2b17b.mp4 media/videos/eval/1_eval_1_ac781709866b92b2b17b.mp4 +2025-01-03 21:25:03,458 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/0_eval_0_27c97977eb157644346f.mp4 media/videos/eval/0_eval_0_27c97977eb157644346f.mp4 +2025-01-03 21:25:03,458 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/4_eval_4_45ef389d8f994db06163.mp4 media/videos/eval/4_eval_4_45ef389d8f994db06163.mp4 +2025-01-03 21:25:03,458 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-03 21:25:03,458 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table/eval/total_success_rate_ci_6_b5c7077c1f7a17f44586.table.json media/table/eval/total_success_rate_ci_6_b5c7077c1f7a17f44586.table.json +2025-01-03 21:25:03,458 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 10 +2025-01-03 21:25:03,458 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:25:03,459 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 21:25:03,461 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:25:03,461 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 21:25:03,461 INFO SenderThread:1976663 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:25:03,643 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/requirements.txt +2025-01-03 21:25:03,645 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/config.yaml +2025-01-03 21:25:03,693 INFO wandb-upload_1:1976663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:25:03,730 INFO wandb-upload_4:1976663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:25:03,734 INFO wandb-upload_3:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/p34cbej2-events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:25:03,816 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:25:03,816 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:25:03,934 INFO Thread-11 (_thread_body):1976663 [sender.py:transition_state():614] send defer: 11 +2025-01-03 21:25:03,934 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:25:03,934 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 21:25:03,935 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:25:03,935 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 21:25:03,935 INFO SenderThread:1976663 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 21:25:03,936 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 12 +2025-01-03 21:25:03,936 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:25:03,936 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 21:25:03,936 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:25:03,936 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 21:25:03,936 INFO SenderThread:1976663 [file_stream.py:finish():601] file stream finish called +2025-01-03 21:25:04,146 INFO SenderThread:1976663 [file_stream.py:finish():605] file stream finish is done +2025-01-03 21:25:04,146 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 13 +2025-01-03 21:25:04,146 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:25:04,146 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 21:25:04,146 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:25:04,147 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 21:25:04,147 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 14 +2025-01-03 21:25:04,147 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:25:04,147 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 21:25:04,147 DEBUG SenderThread:1976663 [sender.py:send():379] send: final +2025-01-03 21:25:04,147 DEBUG SenderThread:1976663 [sender.py:send():379] send: footer +2025-01-03 21:25:04,147 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:25:04,147 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 21:25:04,148 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:25:04,148 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:25:04,148 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 21:25:04,148 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:25:04,148 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 21:25:04,149 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 21:25:04,149 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: server_info +2025-01-03 21:25:04,151 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:25:04,211 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:25:04,211 INFO MainThread:1976663 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 21:25:04,212 INFO MainThread:1976663 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 21:25:04,212 INFO MainThread:1976663 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 21:25:04,212 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 21:25:04,212 INFO HandlerThread:1976663 [handler.py:finish():882] shutting down handler +2025-01-03 21:25:05,149 INFO WriterThread:1976663 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/run-vgz4bwx7.wandb +2025-01-03 21:25:05,211 INFO SenderThread:1976663 [sender.py:finish():1615] shutting down sender +2025-01-03 21:25:05,211 INFO SenderThread:1976663 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:25:05,211 INFO SenderThread:1976663 [file_pusher.py:join():175] waiting for file pusher diff --git a/vgz4bwx7/wandb/debug.log b/vgz4bwx7/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..15319c93c69a922a20a9a14a46c6c757f724b5a8 --- /dev/null +++ b/vgz4bwx7/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 21:20:43,957 INFO MainThread:1976476 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 21:20:43,957 INFO MainThread:1976476 [wandb_setup.py:_flush():76] Configure stats pid to 1976476 +2025-01-03 21:20:43,957 INFO MainThread:1976476 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 21:20:43,957 INFO MainThread:1976476 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/settings +2025-01-03 21:20:43,957 INFO MainThread:1976476 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 21:20:43,957 INFO MainThread:1976476 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 21:20:43,957 INFO MainThread:1976476 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 21:20:43,957 INFO MainThread:1976476 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 21:20:43,957 INFO MainThread:1976476 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/logs/debug.log +2025-01-03 21:20:43,958 INFO MainThread:1976476 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/logs/debug-internal.log +2025-01-03 21:20:43,958 INFO MainThread:1976476 [wandb_init.py:init():569] calling init triggers +2025-01-03 21:20:43,958 INFO MainThread:1976476 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7'} +2025-01-03 21:20:43,958 INFO MainThread:1976476 [wandb_init.py:init():619] starting backend +2025-01-03 21:20:43,958 INFO MainThread:1976476 [wandb_init.py:init():623] setting up manager +2025-01-03 21:20:43,962 INFO MainThread:1976476 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 21:20:43,963 INFO MainThread:1976476 [wandb_init.py:init():631] backend started and connected +2025-01-03 21:20:43,976 INFO MainThread:1976476 [wandb_init.py:init():720] updated telemetry +2025-01-03 21:20:43,985 INFO MainThread:1976476 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 21:20:44,382 INFO MainThread:1976476 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 21:20:44,423 INFO MainThread:1976476 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 21:20:44,424 INFO MainThread:1976476 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 21:20:44,763 INFO MainThread:1976476 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 21:20:44,763 INFO MainThread:1976476 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 21:20:44,763 INFO MainThread:1976476 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 21:20:44,763 INFO MainThread:1976476 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 21:20:44,766 INFO MainThread:1976476 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 21:20:44,766 INFO MainThread:1976476 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/tb, True +2025-01-03 21:20:50,584 INFO MainThread:1976476 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-03 21:25:05,213 WARNING MsgRouterThr:1976476 [router.py:message_loop():77] message_loop has been closed diff --git a/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/code/FISH/eval_robot.py b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/config.yaml b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07ccf1567ad2e77879dbd833ecd6a237932d2c74 --- /dev/null +++ b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1926_1' +wandb_notes: + desc: null + value: 1926_1_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735957243 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/diff.patch b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c97de4502eea91c4c009a6fb51db681afeec762b --- /dev/null +++ b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/diff.patch @@ -0,0 +1,57 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..9234106 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,19 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_2' # seed 2 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json new file mode 100644 index 0000000000000000000000000000000000000000..faa9910a25d3f0826c10a7b81de69b8b94278cd1 --- /dev/null +++ b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5363743901179997, 0, 0.48415964842159465, 0, 0.21277225023292168, 0, 0.1673506349990479]]} \ No newline at end of file diff --git a/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table/eval/total_success_rate_ci_6_b5c7077c1f7a17f44586.table.json b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table/eval/total_success_rate_ci_6_b5c7077c1f7a17f44586.table.json new file mode 100644 index 0000000000000000000000000000000000000000..4a92886fc4ffacef78b2cd88b8e22d57614560a3 --- /dev/null +++ b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table/eval/total_success_rate_ci_6_b5c7077c1f7a17f44586.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[9, 20, 0.45, 0.23819544916620672, 0.6925441622348981, 0.23861089354020815, 0.65288241204542, 0.26172605161663975, 0.6364964097465273, 0.33280161026483923, 0.6040334477812601]]} \ No newline at end of file diff --git a/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/0_eval_0_27c97977eb157644346f.mp4 b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/0_eval_0_27c97977eb157644346f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..85fa7267a8bd5845d43cca89c655f11624aef098 --- /dev/null +++ b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/0_eval_0_27c97977eb157644346f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27c97977eb157644346f1978f0b0b20aee68b2d02d18588969187c8079b2ca5a +size 993819 diff --git a/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/1_eval_1_ac781709866b92b2b17b.mp4 b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/1_eval_1_ac781709866b92b2b17b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ad2d35ef170202fffee6d39640c6ae0de448902a --- /dev/null +++ b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/1_eval_1_ac781709866b92b2b17b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac781709866b92b2b17b061e19df1d9075252fcc5d6b984dfc7757a9483d10e7 +size 601276 diff --git a/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/2_eval_2_dab3f95c8b865ba8aed6.mp4 b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/2_eval_2_dab3f95c8b865ba8aed6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df6b5c55eb41490d1b34d7c0426e33cbcc622639 --- /dev/null +++ b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/2_eval_2_dab3f95c8b865ba8aed6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dab3f95c8b865ba8aed6d5334650c6a14ef5d50ae1807606872a21792c0ae0b7 +size 882000 diff --git a/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/3_eval_3_c216389b31307ee9d9e4.mp4 b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/3_eval_3_c216389b31307ee9d9e4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7a0c7aaba24625679cedf4ace26dfd0e427752ec --- /dev/null +++ b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/3_eval_3_c216389b31307ee9d9e4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c216389b31307ee9d9e40ac7deb7ff69a0fbfc2ea7d9d271856e38bc3eb9bb54 +size 821461 diff --git a/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/4_eval_4_45ef389d8f994db06163.mp4 b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/4_eval_4_45ef389d8f994db06163.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9e4ca776d1180a17d84af4aea1739f28c4e9d013 --- /dev/null +++ b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/4_eval_4_45ef389d8f994db06163.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45ef389d8f994db06163105981839461bce48d04757a4e099c64220d28141532 +size 724143 diff --git a/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..6ecf1b9d91dfae2c8da355ee27f410177b2082ea --- /dev/null +++ b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log @@ -0,0 +1,291 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735957250.584591]: resetting environment +[INFO] [1735957250.589640]: cleared current plan +[INFO] [1735957250.590878]: moving to home +[INFO] [1735957251.593273]: reached home +[INFO] [1735957251.594629]: reset action history +[INFO] [1735957254.303929]: environment reset +Starting episode 0 +[INFO] [1735957254.309125]: resetting environment +[INFO] [1735957254.315711]: cleared current plan +[INFO] [1735957254.317008]: moving to home +[INFO] [1735957255.319849]: reached home +[INFO] [1735957255.321361]: reset action history +[INFO] [1735957258.030396]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735957263.128142]: resetting environment +[INFO] [1735957263.134520]: cleared current plan +[INFO] [1735957263.135488]: moving to home +[INFO] [1735957264.137255]: reached home +[INFO] [1735957264.138502]: reset action history +[INFO] [1735957266.848147]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/episode_rosbags/episode_0_2025-01-03-21-21-06.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1976476.log +started roslaunch server http://158.130.50.37:33857/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1976769] +process[print_text-2]: started with pid [1976793] +process[rosbag_record-3]: started with pid [1976794] +started bagging! +preempting episode +[INFO] [1735957289.258296]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735957289.971775]: Stored episode 1. +[INFO] [1735957289.972456]: Saving video... +[INFO] [1735957290.187891]: Video saved! +Starting episode 1 +[INFO] [1735957290.203419]: resetting environment +[INFO] [1735957290.212437]: cleared current plan +[INFO] [1735957290.212726]: moving to home +[INFO] [1735957293.917260]: reached home +[INFO] [1735957293.917717]: reset action history +[INFO] [1735957296.629634]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735957311.339928]: resetting environment +[INFO] [1735957311.343150]: cleared current plan +[INFO] [1735957311.343299]: moving to home +[INFO] [1735957312.344827]: reached home +[INFO] [1735957312.345263]: reset action history +[INFO] [1735957315.053581]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/episode_rosbags/episode_1_2025-01-03-21-21-55.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1976476.log +started roslaunch server http://158.130.50.37:39391/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1976928] +process[print_text-5]: started with pid [1976929] +process[rosbag_record-6]: started with pid [1976930] +started bagging! +preempting episode +[INFO] [1735957327.720621]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735957328.462594]: Stored episode 2. +[WARN] [1735957328.877273]: Plan exhausted +[WARN] [1735957328.917070]: Plan exhausted +[INFO] [1735957328.462883]: Saving video... +[INFO] [1735957328.888203]: Video saved! +Starting episode 2 +[INFO] [1735957328.914438]: resetting environment +[INFO] [1735957328.930826]: cleared current plan +[INFO] [1735957328.932241]: moving to home +[INFO] [1735957332.635410]: reached home +[INFO] [1735957332.637713]: reset action history +[INFO] [1735957335.411901]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735957351.408730]: resetting environment +[INFO] [1735957351.426311]: cleared current plan +[INFO] [1735957351.426749]: moving to home +[INFO] [1735957352.428941]: reached home +[INFO] [1735957352.429450]: reset action history +[INFO] [1735957355.138708]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/episode_rosbags/episode_2_2025-01-03-21-22-35.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1976476.log +started roslaunch server http://158.130.50.37:43917/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1977083] +process[print_text-8]: started with pid [1977084] +process[rosbag_record-9]: started with pid [1977085] +started bagging! +[WARN] [1735957376.001883]: Plan exhausted +[WARN] [1735957376.035888]: Plan exhausted +[WARN] [1735957376.075385]: Plan exhausted +[WARN] [1735957376.115037]: Plan exhausted +[WARN] [1735957376.154608]: Plan exhausted +[WARN] [1735957376.194962]: Plan exhausted +preempting episode +[INFO] [1735957374.640077]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1735957375.491335]: Stored episode 3. +[INFO] [1735957375.493085]: Saving video... +[INFO] [1735957376.179807]: Video saved! +Starting episode 3 +[INFO] [1735957376.220358]: resetting environment +[INFO] [1735957376.230216]: cleared current plan +[INFO] [1735957376.230775]: moving to home +[INFO] [1735957379.933134]: reached home +[INFO] [1735957379.944436]: reset action history +[INFO] [1735957382.894664]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735957401.344015]: resetting environment +[INFO] [1735957401.373357]: cleared current plan +[INFO] [1735957401.380577]: moving to home +[INFO] [1735957402.382907]: reached home +[INFO] [1735957402.389047]: reset action history +[INFO] [1735957405.158730]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/episode_rosbags/episode_3_2025-01-03-21-23-25.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1976476.log +started roslaunch server http://158.130.50.37:45515/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1977223] +process[print_text-11]: started with pid [1977224] +process[rosbag_record-12]: started with pid [1977248] +started bagging! +For topic gripper_width: timestamp difference is 101077302 for nearest: 1735957415528237088 - target: 1735957415427159786 at idx 153 +For topic gripper_width: timestamp difference is 106551850 for nearest: 1735957422328237736 - target: 1735957422221685886 at idx 246 +[WARN] [1735957424.416142]: Plan exhausted +[WARN] [1735957424.458561]: Plan exhausted +preempting episode +[INFO] [1735957423.164149]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1735957424.052248]: Stored episode 4. +[INFO] [1735957424.058512]: Saving video... +[WARN] [1735957424.498900]: Plan exhausted +[WARN] [1735957424.538774]: Plan exhausted +[WARN] [1735957424.575601]: Plan exhausted +[WARN] [1735957424.618751]: Plan exhausted +[WARN] [1735957424.655681]: Plan exhausted +[WARN] [1735957424.696148]: Plan exhausted +[WARN] [1735957424.736300]: Plan exhausted +[WARN] [1735957424.775556]: Plan exhausted +[INFO] [1735957424.708302]: Video saved! +Starting episode 4 +[INFO] [1735957424.751323]: resetting environment +[INFO] [1735957424.783352]: cleared current plan +[INFO] [1735957424.783985]: moving to home +[INFO] [1735957428.388953]: reached home +[INFO] [1735957428.393404]: reset action history +[INFO] [1735957431.143394]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735957446.163055]: resetting environment +[INFO] [1735957446.200680]: cleared current plan +[INFO] [1735957446.208840]: moving to home +[INFO] [1735957447.215898]: reached home +[INFO] [1735957447.222270]: reset action history +[INFO] [1735957450.011757]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/episode_rosbags/episode_4_2025-01-03-21-24-10.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1976476.log +started roslaunch server http://158.130.50.37:46625/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1977399] +process[print_text-14]: started with pid [1977400] +process[rosbag_record-15]: started with pid [1977401] +started bagging! +preempting episode +[INFO] [1735957465.820748]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[WARN] [1735957466.936614]: Plan exhausted +[WARN] [1735957466.976199]: Plan exhausted +[WARN] [1735957467.015553]: Plan exhausted +[INFO] [1735957466.709870]: Stored episode 5. +[INFO] [1735957466.718580]: Saving video... +[INFO] [1735957466.940382]: Video saved! +[INFO] [1735957466.998619]: resetting environment +[INFO] [1735957467.016077]: cleared current plan +[INFO] [1735957467.016460]: moving to home +[INFO] [1735957470.620845]: reached home +[INFO] [1735957470.630990]: reset action history +[INFO] [1735957473.513177]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/requirements.txt b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-metadata.json b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..34b69232234f3419b92487dda1a8da04d7abb40d --- /dev/null +++ b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T02:20:44.447582", + "startedAt": "2025-01-04T02:20:43.954878", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3936.0048750000005, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.968, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4498.707, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4497.939, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4496.57, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.962, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 812.3028678894043 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..3756ccd51935468ea06652aba06f11fbc8cd345a --- /dev/null +++ b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "27c97977eb157644346f1978f0b0b20aee68b2d02d18588969187c8079b2ca5a", "size": 993819, "path": "media/videos/eval/0_eval_0_27c97977eb157644346f.mp4"}, "global_step": 331, "_timestamp": 1735957490.389659, "_runtime": 246.42569303512573, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "ac781709866b92b2b17b061e19df1d9075252fcc5d6b984dfc7757a9483d10e7", "size": 601276, "path": "media/videos/eval/1_eval_1_ac781709866b92b2b17b.mp4"}, "eval/num_success": 0.0, "episode": 20.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5363743901252747, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4841596484184265, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.21277225017547607, "eval/success_rate_lb_0.1": 0.0, "eval/success_rate_ub_0.1": 0.167350634932518, "eval/2_eval": {"_type": "video-file", "sha256": "dab3f95c8b865ba8aed6d5334650c6a14ef5d50ae1807606872a21792c0ae0b7", "size": 882000, "path": "media/videos/eval/2_eval_2_dab3f95c8b865ba8aed6.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "c216389b31307ee9d9e40ac7deb7ff69a0fbfc2ea7d9d271856e38bc3eb9bb54", "size": 821461, "path": "media/videos/eval/3_eval_3_c216389b31307ee9d9e4.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "45ef389d8f994db06163105981839461bce48d04757a4e099c64220d28141532", "size": 724143, "path": "media/videos/eval/4_eval_4_45ef389d8f994db06163.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "e6508f0181aa84573747f0d380a2c6725005451ce9eb7fcdf77e1f39c90f7528", "size": 332, "artifact_path": "wandb-client-artifact://txwmv0nfnze9yi41o52zk8sx55duw8yved308qorkryo9dlsg7ub5utgnvrr92klfw1d3lpus28we7tkrq4sqvygucnwh96magnoqhvprja0a4gr1bvl34qbkmnn2t1r/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://xvd65gwfvv46ptv5hplzh6t3n9vcpr723nw3f1ir6rziifuc7gf92tkpmz88so8381dhmwo8cy6hk0bwbjgta1adioim8iva0g2i0w34lnqm5qrhumdjj5c32107t0xa:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 9, "total_num_episodes": 20, "total_success_rate": 0.45, "total_success_rate_lb_0.01": 0.23819544916620672, "total_success_rate_ub_0.01": 0.6925441622348981, "total_success_rate_lb_0.025": 0.23861089354020815, "total_success_rate_ub_0.025": 0.65288241204542, "total_success_rate_lb_0.05": 0.26172605161663975, "total_success_rate_ub_0.05": 0.6364964097465273, "total_success_rate_lb_0.1": 0.33280161026483923, "total_success_rate_ub_0.1": 0.6040334477812601, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "b5c7077c1f7a17f4458666241340ed1c606ef9c62bbd81c31a82b2e661e6a7b2", "size": 507, "artifact_path": "wandb-client-artifact://x8gmwv71puf4y2ejhmbvf3tzltq9xrezxtygjprtx6pq8uf7vuymdm9kxsa0kgp49mexx3k0n0emic11cy7pniq1czjkj8o3dpvuy5u30q2bfwcil9tfbegvvbefrv9x/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://6pugho9ol620rqc5xh0lgt6o01kvd9lub8myb3p58fg5s9vvpknf7clo9nynzn98l7iamwpna04psb7p1yl0y6ytkgg9c0ijx900xg2bbd6i57x9otgiqgdvcovrw1xx:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_b5c7077c1f7a17f44586.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 248}, "eval/total_success_rate": 0.44999998807907104, "eval/total_success_rate_lb_0.01": 0.23819544911384583, "eval/total_success_rate_ub_0.01": 0.692544162273407, "eval/total_success_rate_lb_0.025": 0.2386108934879303, "eval/total_success_rate_ub_0.025": 0.6528823971748352, "eval/total_success_rate_lb_0.05": 0.261726051568985, "eval/total_success_rate_ub_0.05": 0.6364964246749878, "eval/total_success_rate_lb_0.1": 0.33280161023139954, "eval/total_success_rate_ub_0.1": 0.6040334701538086} \ No newline at end of file diff --git a/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/logs/debug-internal.log b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4d1660103d9c4e3c708245ab57de53078c8071bd --- /dev/null +++ b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/logs/debug-internal.log @@ -0,0 +1,737 @@ +2025-01-03 21:20:43,964 INFO StreamThr :1976663 [internal.py:wandb_internal():85] W&B internal server running at pid: 1976663, started at: 2025-01-03 21:20:43.963679 +2025-01-03 21:20:43,966 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status +2025-01-03 21:20:43,977 INFO WriterThread:1976663 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/run-vgz4bwx7.wandb +2025-01-03 21:20:43,977 DEBUG SenderThread:1976663 [sender.py:send():379] send: header +2025-01-03 21:20:43,990 DEBUG SenderThread:1976663 [sender.py:send():379] send: run +2025-01-03 21:20:44,366 INFO SenderThread:1976663 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files +2025-01-03 21:20:44,366 INFO SenderThread:1976663 [sender.py:_start_run_threads():1188] run started: vgz4bwx7 with start time 1735957243.963966 +2025-01-03 21:20:44,383 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 21:20:44,383 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: check_version +2025-01-03 21:20:44,429 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 21:20:44,434 DEBUG HandlerThread:1976663 [system_info.py:__init__():26] System info init +2025-01-03 21:20:44,434 DEBUG HandlerThread:1976663 [system_info.py:__init__():41] System info init done +2025-01-03 21:20:44,434 INFO HandlerThread:1976663 [system_monitor.py:start():194] Starting system monitor +2025-01-03 21:20:44,434 INFO SystemMonitor:1976663 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 21:20:44,435 INFO HandlerThread:1976663 [system_monitor.py:probe():214] Collecting system info +2025-01-03 21:20:44,435 INFO SystemMonitor:1976663 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 21:20:44,436 INFO SystemMonitor:1976663 [interfaces.py:start():188] Started disk monitoring +2025-01-03 21:20:44,436 INFO SystemMonitor:1976663 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 21:20:44,437 INFO SystemMonitor:1976663 [interfaces.py:start():188] Started memory monitoring +2025-01-03 21:20:44,437 INFO SystemMonitor:1976663 [interfaces.py:start():188] Started network monitoring +2025-01-03 21:20:44,447 DEBUG HandlerThread:1976663 [system_info.py:probe():152] Probing system +2025-01-03 21:20:44,448 DEBUG HandlerThread:1976663 [system_info.py:_probe_git():137] Probing git +2025-01-03 21:20:44,453 DEBUG HandlerThread:1976663 [system_info.py:_probe_git():145] Probing git done +2025-01-03 21:20:44,453 DEBUG HandlerThread:1976663 [system_info.py:probe():200] Probing system done +2025-01-03 21:20:44,453 DEBUG HandlerThread:1976663 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T02:20:44.447582', 'startedAt': '2025-01-04T02:20:43.954878', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3936.0048750000005, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.968, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.707, 'min': 800.0, 'max': 4700.0}, {'current': 4497.939, 'min': 800.0, 'max': 4700.0}, {'current': 4496.57, 'min': 800.0, 'max': 4700.0}, {'current': 4499.962, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 812.3028678894043}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 21:20:44,453 INFO HandlerThread:1976663 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 21:20:44,453 INFO HandlerThread:1976663 [system_monitor.py:probe():227] Publishing system info +2025-01-03 21:20:44,453 DEBUG HandlerThread:1976663 [system_info.py:_save_code():44] Saving code +2025-01-03 21:20:44,455 DEBUG HandlerThread:1976663 [system_info.py:_save_code():67] Saving code done +2025-01-03 21:20:44,455 DEBUG HandlerThread:1976663 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 21:20:44,518 DEBUG HandlerThread:1976663 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 21:20:44,518 INFO HandlerThread:1976663 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 21:20:44,522 DEBUG SenderThread:1976663 [sender.py:send():379] send: files +2025-01-03 21:20:44,522 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 21:20:44,522 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 21:20:44,522 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 21:20:44,762 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 21:20:44,762 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: python_packages +2025-01-03 21:20:44,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:20:44,764 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:20:44,804 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/h94zdgtm-diff.patch +2025-01-03 21:20:44,805 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:44,805 INFO HandlerThread:1976663 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/tb" + save: true +} +control { + relay_id: "90088d12147245b2a5b6f7014b49871b" +} +_info { + stream_id: "vgz4bwx7" +} + +2025-01-03 21:20:44,805 DEBUG HandlerThread:1976663 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 21:20:44,814 INFO wandb-upload_1:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/un2iepj0-code/FISH/eval_robot.py +2025-01-03 21:20:44,818 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/qiejtn4c-wandb-metadata.json +2025-01-03 21:20:44,865 DEBUG SenderThread:1976663 [sender.py:send():379] send: telemetry +2025-01-03 21:20:44,874 DEBUG SenderThread:1976663 [sender.py:send():379] send: tbrecord +2025-01-03 21:20:44,875 DEBUG SenderThread:1976663 [sender.py:send():379] send: files +2025-01-03 21:20:44,875 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 with policy live +2025-01-03 21:20:45,058 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/mehq0b1b-events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:20:45,368 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/requirements.txt +2025-01-03 21:20:45,368 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:20:45,368 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:20:45,368 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-metadata.json +2025-01-03 21:20:45,368 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/diff.patch +2025-01-03 21:20:45,368 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/code/FISH/eval_robot.py +2025-01-03 21:20:45,369 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/code/FISH +2025-01-03 21:20:45,369 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/code +2025-01-03 21:20:45,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:46,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:47,367 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:20:47,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:48,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:49,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:20:49,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:50,584 DEBUG SenderThread:1976663 [sender.py:send():379] send: config +2025-01-03 21:20:50,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:51,368 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:20:51,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:52,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:53,369 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:20:53,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:54,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:20:54,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:55,369 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:20:55,765 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:56,765 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:57,370 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:20:57,765 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:58,765 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:59,371 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:20:59,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:20:59,763 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:20:59,805 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:59,847 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:00,765 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:01,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:02,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:03,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:04,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:05,372 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:21:05,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:05,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:06,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:07,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:08,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:09,373 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:21:09,767 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:10,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:10,767 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:11,767 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:12,767 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:13,767 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:14,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:21:14,763 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:21:14,805 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:15,768 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:15,828 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:16,375 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/config.yaml +2025-01-03 21:21:16,768 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:17,768 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:18,768 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:19,768 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:20,768 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:21,765 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:21,769 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:22,769 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:23,769 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:24,769 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:25,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:26,765 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:26,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:27,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:28,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:29,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:21:29,763 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:21:29,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:30,202 DEBUG SenderThread:1976663 [sender.py:send():379] send: files +2025-01-03 21:21:30,202 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_27c97977eb157644346f.mp4 with policy now +2025-01-03 21:21:30,202 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:21:30,203 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:21:30,203 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:21:30,204 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:21:30,379 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:21:30,379 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/0_eval_0_27c97977eb157644346f.mp4 +2025-01-03 21:21:30,379 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media +2025-01-03 21:21:30,379 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos +2025-01-03 21:21:30,379 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval +2025-01-03 21:21:30,424 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/65hkwpps-media/videos/eval/0_eval_0_27c97977eb157644346f.mp4 +2025-01-03 21:21:30,676 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/p1uuy1s5-media/videos/eval/0_eval_0_27c97977eb157644346f.mp4 +2025-01-03 21:21:30,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:31,379 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:21:31,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:32,213 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:32,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:33,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:34,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:35,381 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:21:35,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:36,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:37,673 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:37,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:38,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:39,382 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:21:39,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:40,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:41,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:42,765 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:42,772 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:43,772 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:44,437 DEBUG SystemMonitor:1976663 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 21:21:44,438 DEBUG SenderThread:1976663 [sender.py:send():379] send: stats +2025-01-03 21:21:44,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:21:44,764 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:21:44,805 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:45,772 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:46,772 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:47,772 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:47,919 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:48,773 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:49,773 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:50,773 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:51,386 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:21:51,618 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/tynca6yv-events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:21:51,773 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:52,773 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:53,385 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:53,387 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:21:53,773 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:54,773 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:55,773 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:56,774 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:57,388 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:21:57,774 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:58,774 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:21:59,305 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:21:59,763 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:21:59,764 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:21:59,805 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:00,774 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:01,774 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:02,774 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:03,774 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:04,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:04,774 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:05,775 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:06,775 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:07,775 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:08,778 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:08,921 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:22:08,921 DEBUG SenderThread:1976663 [sender.py:send():379] send: files +2025-01-03 21:22:08,921 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_ac781709866b92b2b17b.mp4 with policy now +2025-01-03 21:22:08,931 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:22:08,931 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:22:08,932 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:22:09,132 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/ag7wzq5w-media/videos/eval/1_eval_1_ac781709866b92b2b17b.mp4 +2025-01-03 21:22:09,392 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:22:09,392 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:22:09,392 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/1_eval_1_ac781709866b92b2b17b.mp4 +2025-01-03 21:22:09,392 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval +2025-01-03 21:22:09,647 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/hxupvmqv-media/videos/eval/1_eval_1_ac781709866b92b2b17b.mp4 +2025-01-03 21:22:09,777 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:09,780 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:10,777 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:11,393 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:22:11,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:12,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:13,779 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:14,439 DEBUG SenderThread:1976663 [sender.py:send():379] send: stats +2025-01-03 21:22:14,769 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:22:14,769 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:22:14,813 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:14,834 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:15,394 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:22:15,779 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:16,779 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:17,395 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:22:17,779 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:18,779 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:19,779 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:20,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:20,780 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:21,780 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:22,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:23,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:24,780 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:25,770 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:25,780 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:26,780 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:27,780 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:28,780 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:29,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:22:29,765 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:22:29,805 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:30,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:30,876 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:31,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:32,245 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:22:32,245 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:22:32,246 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:22:32,400 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:22:32,400 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:22:32,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:33,400 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:22:33,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:34,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:35,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:36,404 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:36,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:37,401 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:22:37,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:38,781 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:39,782 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:40,782 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:41,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:41,782 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:42,782 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:43,782 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:44,439 DEBUG SenderThread:1976663 [sender.py:send():379] send: stats +2025-01-03 21:22:44,764 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:22:44,765 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:22:44,805 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:45,784 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:46,785 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:46,839 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:47,784 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:48,784 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:49,785 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:50,784 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:51,785 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:52,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:52,785 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:53,785 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:54,785 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:55,795 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:56,215 DEBUG SenderThread:1976663 [sender.py:send():379] send: files +2025-01-03 21:22:56,215 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_dab3f95c8b865ba8aed6.mp4 with policy now +2025-01-03 21:22:56,218 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:22:56,218 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:22:56,218 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:22:56,219 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:22:56,408 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:22:56,408 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/2_eval_2_dab3f95c8b865ba8aed6.mp4 +2025-01-03 21:22:56,408 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval +2025-01-03 21:22:56,426 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/pixxa322-media/videos/eval/2_eval_2_dab3f95c8b865ba8aed6.mp4 +2025-01-03 21:22:56,708 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/j7tl2ri7-media/videos/eval/2_eval_2_dab3f95c8b865ba8aed6.mp4 +2025-01-03 21:22:56,789 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:57,408 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:22:57,793 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:58,231 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:22:58,791 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:22:59,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:22:59,766 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:22:59,809 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:00,792 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:01,409 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:01,790 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:02,881 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:03,879 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:03,941 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:04,875 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:05,411 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:05,878 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:06,878 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:07,892 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:08,880 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:09,775 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:09,881 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:10,881 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:11,886 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:12,885 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:13,888 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:14,439 DEBUG SenderThread:1976663 [sender.py:send():379] send: stats +2025-01-03 21:23:14,766 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:23:14,766 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:23:14,883 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:14,889 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:15,885 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:16,887 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:17,891 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:18,884 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:19,887 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:20,779 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:20,887 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:21,416 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:23:21,659 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/y6506mdo-events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:23:21,889 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:22,093 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:23:22,093 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:23:22,093 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:23:22,416 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:23:22,887 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:23,416 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:23,887 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:24,891 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:25,888 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:26,429 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:26,888 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:27,418 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:27,888 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:28,888 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:29,765 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:23:29,765 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:23:29,888 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:30,888 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:31,888 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:31,906 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:32,889 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:33,889 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:34,889 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:35,889 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:36,889 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:37,421 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:37,585 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:37,889 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:38,890 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:39,890 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:40,890 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:41,890 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:42,890 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:43,158 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:43,423 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:43,892 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:44,440 DEBUG SenderThread:1976663 [sender.py:send():379] send: stats +2025-01-03 21:23:44,739 DEBUG SenderThread:1976663 [sender.py:send():379] send: files +2025-01-03 21:23:44,739 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c216389b31307ee9d9e4.mp4 with policy now +2025-01-03 21:23:44,744 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:23:44,744 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:23:44,745 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:23:44,745 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:23:44,775 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:23:44,775 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:23:44,890 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:44,981 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/m8fqdeit-media/videos/eval/3_eval_3_c216389b31307ee9d9e4.mp4 +2025-01-03 21:23:45,424 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:45,424 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:23:45,424 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/3_eval_3_c216389b31307ee9d9e4.mp4 +2025-01-03 21:23:45,425 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval +2025-01-03 21:23:45,671 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/1bl9vy5b-media/videos/eval/3_eval_3_c216389b31307ee9d9e4.mp4 +2025-01-03 21:23:45,897 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:46,896 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:47,425 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:47,895 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:48,391 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:48,902 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:49,426 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:49,900 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:50,912 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:51,896 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:52,899 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:53,427 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:23:53,903 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:54,185 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:23:54,908 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:55,901 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:56,901 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:57,905 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:58,912 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:59,772 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:23:59,772 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:23:59,901 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:23:59,904 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:00,912 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:01,906 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:02,911 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:03,912 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:04,909 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:05,792 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:05,909 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:06,151 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:24:06,151 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:06,151 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:06,431 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:24:06,431 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:24:06,911 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:07,432 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:07,914 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:08,913 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:09,432 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:09,915 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:10,913 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:11,433 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:11,468 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:11,913 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:12,913 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:13,914 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:14,440 DEBUG SenderThread:1976663 [sender.py:send():379] send: stats +2025-01-03 21:24:14,769 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:24:14,769 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:24:14,913 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:15,915 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:16,857 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:16,915 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:17,915 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:18,915 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:19,916 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:20,916 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:21,916 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:22,789 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:22,916 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:23,916 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:24,916 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:25,917 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:26,942 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:27,025 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:24:27,026 DEBUG SenderThread:1976663 [sender.py:send():379] send: files +2025-01-03 21:24:27,026 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_45ef389d8f994db06163.mp4 with policy now +2025-01-03 21:24:27,026 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:24:27,026 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:27,027 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:27,233 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/gvz5rqpa-media/videos/eval/4_eval_4_45ef389d8f994db06163.mp4 +2025-01-03 21:24:27,439 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:27,439 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:24:27,439 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/4_eval_4_45ef389d8f994db06163.mp4 +2025-01-03 21:24:27,439 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval +2025-01-03 21:24:27,684 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/2abo3b9s-media/videos/eval/4_eval_4_45ef389d8f994db06163.mp4 +2025-01-03 21:24:27,945 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:28,028 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:28,944 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:29,439 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:29,779 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:24:29,779 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:24:29,944 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:30,952 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:31,440 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:31,943 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:32,945 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:33,441 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:33,514 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:33,946 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:34,959 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:35,441 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:35,945 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:36,951 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:37,953 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:38,797 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:38,951 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:39,956 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:40,961 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:41,957 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:42,963 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:43,960 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:44,496 DEBUG SenderThread:1976663 [sender.py:send():379] send: stats +2025-01-03 21:24:44,497 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:44,771 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:24:44,771 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:24:44,963 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:45,966 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:46,966 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:47,968 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:48,967 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:49,446 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:24:49,692 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/x7c1goys-events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:24:49,793 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:49,858 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 21:24:49,858 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 21:24:49,964 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:50,120 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:24:50,316 INFO wandb-upload_2:1976663 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpoe0_7d8x +2025-01-03 21:24:50,446 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:24:50,447 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-03 21:24:50,447 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media +2025-01-03 21:24:50,447 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table +2025-01-03 21:24:50,447 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table/eval +2025-01-03 21:24:50,491 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 21:24:50,503 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:24:50,729 INFO SenderThread:1976663 [sender.py:send_request_log_artifact():1525] logged artifact run-vgz4bwx7-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzkwOTY0NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3ODkzNg==', 'latestArtifact': None}} +2025-01-03 21:24:50,729 DEBUG SenderThread:1976663 [sender.py:send():379] send: files +2025-01-03 21:24:50,729 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json with policy now +2025-01-03 21:24:50,729 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:24:50,729 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,730 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,730 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:24:50,730 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,734 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,734 DEBUG SenderThread:1976663 [sender.py:send():379] send: telemetry +2025-01-03 21:24:50,734 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,734 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,734 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,735 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,735 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,735 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,735 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,736 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,736 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,736 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,736 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,736 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,737 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,737 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,737 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,737 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,738 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,738 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,738 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,738 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,739 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:50,739 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:50,739 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 21:24:50,912 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/9lxpfd20-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-03 21:24:50,962 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:51,144 INFO wandb-upload_1:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/s201i0pv-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-03 21:24:51,150 INFO wandb-upload_2:1976663 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4hy42nnt +2025-01-03 21:24:51,447 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:51,447 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:24:51,447 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table/eval/total_success_rate_ci_6_b5c7077c1f7a17f44586.table.json +2025-01-03 21:24:51,447 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table/eval +2025-01-03 21:24:51,593 INFO SenderThread:1976663 [sender.py:send_request_log_artifact():1525] logged artifact run-vgz4bwx7-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzkwOTY3NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3ODkzOQ==', 'latestArtifact': None}} +2025-01-03 21:24:51,593 DEBUG SenderThread:1976663 [sender.py:send():379] send: files +2025-01-03 21:24:51,593 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_b5c7077c1f7a17f44586.table.json with policy now +2025-01-03 21:24:51,594 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:24:51,594 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:51,594 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:51,810 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/jqv5cze4-media/table/eval/total_success_rate_ci_6_b5c7077c1f7a17f44586.table.json +2025-01-03 21:24:51,962 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:24:52,098 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/b3c8flsg-media/table/eval/total_success_rate_ci_6_b5c7077c1f7a17f44586.table.json +2025-01-03 21:24:52,447 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:24:52,813 DEBUG SenderThread:1976663 [sender.py:send():379] send: exit +2025-01-03 21:24:52,813 INFO SenderThread:1976663 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 21:24:52,814 INFO SenderThread:1976663 [sender.py:send_exit():588] handling runtime: 248 +2025-01-03 21:24:52,814 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:52,814 INFO SenderThread:1976663 [sender.py:send_exit():594] send defer +2025-01-03 21:24:52,814 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:24:52,814 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 21:24:52,814 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:24:52,814 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 21:24:52,815 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 1 +2025-01-03 21:24:52,815 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:24:52,815 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 21:24:52,815 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:24:52,815 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 21:24:52,815 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 2 +2025-01-03 21:24:52,815 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:24:52,815 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 21:24:52,815 INFO HandlerThread:1976663 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 21:24:52,815 DEBUG SystemMonitor:1976663 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 21:24:52,816 DEBUG SystemMonitor:1976663 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 21:24:52,817 INFO HandlerThread:1976663 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 21:24:52,817 INFO HandlerThread:1976663 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 21:24:52,868 INFO HandlerThread:1976663 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 21:24:52,868 INFO HandlerThread:1976663 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 21:24:52,869 INFO HandlerThread:1976663 [interfaces.py:finish():200] Joined network monitor +2025-01-03 21:24:52,869 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:24:52,869 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 21:24:52,869 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 3 +2025-01-03 21:24:52,869 DEBUG SenderThread:1976663 [sender.py:send():379] send: stats +2025-01-03 21:24:52,869 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:24:52,869 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 21:24:52,870 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:24:52,870 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 21:24:52,870 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 4 +2025-01-03 21:24:52,870 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:24:52,870 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 21:24:53,448 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:53,448 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:24:55,449 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:24:59,211 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:24:59,211 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:59,211 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 21:24:59,211 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:24:59,212 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 21:24:59,212 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 5 +2025-01-03 21:24:59,212 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:24:59,213 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:24:59,213 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 21:24:59,213 DEBUG SenderThread:1976663 [sender.py:send():379] send: history +2025-01-03 21:24:59,213 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:24:59,214 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:59,214 DEBUG SenderThread:1976663 [sender.py:send():379] send: summary +2025-01-03 21:24:59,214 INFO SenderThread:1976663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:24:59,214 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:24:59,214 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 21:24:59,214 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 6 +2025-01-03 21:24:59,215 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:24:59,215 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 21:24:59,215 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:24:59,215 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 21:24:59,218 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:24:59,358 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 7 +2025-01-03 21:24:59,358 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:24:59,358 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 21:24:59,358 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:24:59,358 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 21:24:59,450 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:24:59,450 INFO Thread-12 :1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/config.yaml +2025-01-03 21:24:59,815 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:25:03,041 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 8 +2025-01-03 21:25:03,042 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:25:03,042 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:25:03,042 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 21:25:03,042 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:25:03,043 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 21:25:03,043 INFO SenderThread:1976663 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 21:25:03,048 INFO SenderThread:1976663 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 21:25:03,051 INFO SenderThread:1976663 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 21:25:03,062 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 9 +2025-01-03 21:25:03,062 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:25:03,062 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 21:25:03,062 DEBUG SenderThread:1976663 [sender.py:send():379] send: artifact +2025-01-03 21:25:03,265 INFO SenderThread:1976663 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzkwNDU3OA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzkwNDU3OA==', 'versionIndex': 410}}} +2025-01-03 21:25:03,265 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:25:03,265 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 21:25:03,266 INFO SenderThread:1976663 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 21:25:03,451 INFO SenderThread:1976663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:25:03,452 INFO SenderThread:1976663 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files +2025-01-03 21:25:03,452 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/config.yaml config.yaml +2025-01-03 21:25:03,452 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json wandb-summary.json +2025-01-03 21:25:03,452 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/requirements.txt requirements.txt +2025-01-03 21:25:03,452 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-metadata.json wandb-metadata.json +2025-01-03 21:25:03,452 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:25:03,452 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log output.log +2025-01-03 21:25:03,452 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/diff.patch diff.patch +2025-01-03 21:25:03,454 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 21:25:03,457 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/2_eval_2_dab3f95c8b865ba8aed6.mp4 media/videos/eval/2_eval_2_dab3f95c8b865ba8aed6.mp4 +2025-01-03 21:25:03,457 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/3_eval_3_c216389b31307ee9d9e4.mp4 media/videos/eval/3_eval_3_c216389b31307ee9d9e4.mp4 +2025-01-03 21:25:03,457 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/1_eval_1_ac781709866b92b2b17b.mp4 media/videos/eval/1_eval_1_ac781709866b92b2b17b.mp4 +2025-01-03 21:25:03,458 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/0_eval_0_27c97977eb157644346f.mp4 media/videos/eval/0_eval_0_27c97977eb157644346f.mp4 +2025-01-03 21:25:03,458 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/videos/eval/4_eval_4_45ef389d8f994db06163.mp4 media/videos/eval/4_eval_4_45ef389d8f994db06163.mp4 +2025-01-03 21:25:03,458 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-03 21:25:03,458 INFO SenderThread:1976663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/media/table/eval/total_success_rate_ci_6_b5c7077c1f7a17f44586.table.json media/table/eval/total_success_rate_ci_6_b5c7077c1f7a17f44586.table.json +2025-01-03 21:25:03,458 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 10 +2025-01-03 21:25:03,458 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:25:03,459 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 21:25:03,461 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:25:03,461 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 21:25:03,461 INFO SenderThread:1976663 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:25:03,643 INFO wandb-upload_2:1976663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/requirements.txt +2025-01-03 21:25:03,645 INFO wandb-upload_0:1976663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/config.yaml +2025-01-03 21:25:03,693 INFO wandb-upload_1:1976663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/wandb-summary.json +2025-01-03 21:25:03,730 INFO wandb-upload_4:1976663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/files/output.log +2025-01-03 21:25:03,734 INFO wandb-upload_3:1976663 [upload_job.py:push():130] Uploaded file /tmp/tmprau2ko2ywandb/p34cbej2-events.out.tfevents.1735957244.leonmkim-ROG-Strix-G15CS-G15CS.1976476.0 +2025-01-03 21:25:03,816 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:25:03,816 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:25:03,934 INFO Thread-11 (_thread_body):1976663 [sender.py:transition_state():614] send defer: 11 +2025-01-03 21:25:03,934 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:25:03,934 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 21:25:03,935 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:25:03,935 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 21:25:03,935 INFO SenderThread:1976663 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 21:25:03,936 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 12 +2025-01-03 21:25:03,936 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:25:03,936 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 21:25:03,936 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:25:03,936 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 21:25:03,936 INFO SenderThread:1976663 [file_stream.py:finish():601] file stream finish called +2025-01-03 21:25:04,146 INFO SenderThread:1976663 [file_stream.py:finish():605] file stream finish is done +2025-01-03 21:25:04,146 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 13 +2025-01-03 21:25:04,146 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:25:04,146 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 21:25:04,146 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:25:04,147 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 21:25:04,147 INFO SenderThread:1976663 [sender.py:transition_state():614] send defer: 14 +2025-01-03 21:25:04,147 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:25:04,147 INFO HandlerThread:1976663 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 21:25:04,147 DEBUG SenderThread:1976663 [sender.py:send():379] send: final +2025-01-03 21:25:04,147 DEBUG SenderThread:1976663 [sender.py:send():379] send: footer +2025-01-03 21:25:04,147 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: defer +2025-01-03 21:25:04,147 INFO SenderThread:1976663 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 21:25:04,148 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:25:04,148 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:25:04,148 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 21:25:04,148 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:25:04,148 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 21:25:04,149 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 21:25:04,149 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: server_info +2025-01-03 21:25:04,151 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:25:04,211 DEBUG SenderThread:1976663 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:25:04,211 INFO MainThread:1976663 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 21:25:04,212 INFO MainThread:1976663 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 21:25:04,212 INFO MainThread:1976663 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 21:25:04,212 DEBUG HandlerThread:1976663 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 21:25:04,212 INFO HandlerThread:1976663 [handler.py:finish():882] shutting down handler +2025-01-03 21:25:05,149 INFO WriterThread:1976663 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/run-vgz4bwx7.wandb +2025-01-03 21:25:05,211 INFO SenderThread:1976663 [sender.py:finish():1615] shutting down sender +2025-01-03 21:25:05,211 INFO SenderThread:1976663 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:25:05,211 INFO SenderThread:1976663 [file_pusher.py:join():175] waiting for file pusher diff --git a/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/logs/debug.log b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..15319c93c69a922a20a9a14a46c6c757f724b5a8 --- /dev/null +++ b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 21:20:43,957 INFO MainThread:1976476 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 21:20:43,957 INFO MainThread:1976476 [wandb_setup.py:_flush():76] Configure stats pid to 1976476 +2025-01-03 21:20:43,957 INFO MainThread:1976476 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 21:20:43,957 INFO MainThread:1976476 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/settings +2025-01-03 21:20:43,957 INFO MainThread:1976476 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 21:20:43,957 INFO MainThread:1976476 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 21:20:43,957 INFO MainThread:1976476 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 21:20:43,957 INFO MainThread:1976476 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 21:20:43,957 INFO MainThread:1976476 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/logs/debug.log +2025-01-03 21:20:43,958 INFO MainThread:1976476 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/logs/debug-internal.log +2025-01-03 21:20:43,958 INFO MainThread:1976476 [wandb_init.py:init():569] calling init triggers +2025-01-03 21:20:43,958 INFO MainThread:1976476 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7'} +2025-01-03 21:20:43,958 INFO MainThread:1976476 [wandb_init.py:init():619] starting backend +2025-01-03 21:20:43,958 INFO MainThread:1976476 [wandb_init.py:init():623] setting up manager +2025-01-03 21:20:43,962 INFO MainThread:1976476 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 21:20:43,963 INFO MainThread:1976476 [wandb_init.py:init():631] backend started and connected +2025-01-03 21:20:43,976 INFO MainThread:1976476 [wandb_init.py:init():720] updated telemetry +2025-01-03 21:20:43,985 INFO MainThread:1976476 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 21:20:44,382 INFO MainThread:1976476 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 21:20:44,423 INFO MainThread:1976476 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 21:20:44,424 INFO MainThread:1976476 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 21:20:44,763 INFO MainThread:1976476 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 21:20:44,763 INFO MainThread:1976476 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 21:20:44,763 INFO MainThread:1976476 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 21:20:44,763 INFO MainThread:1976476 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 21:20:44,766 INFO MainThread:1976476 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 21:20:44,766 INFO MainThread:1976476 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/vgz4bwx7/tb, True +2025-01-03 21:20:50,584 INFO MainThread:1976476 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-03 21:25:05,213 WARNING MsgRouterThr:1976476 [router.py:message_loop():77] message_loop has been closed diff --git a/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/run-vgz4bwx7.wandb b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/run-vgz4bwx7.wandb new file mode 100644 index 0000000000000000000000000000000000000000..2105f500bf5c310c5fb10166e876ddc5872fd47b Binary files /dev/null and b/vgz4bwx7/wandb/run-20250103_212043-vgz4bwx7/run-vgz4bwx7.wandb differ