diff --git a/.gitattributes b/.gitattributes index 659783059b21d82677d9af36d81b459688b5698c..cda2eeb5cd7d1d5d0041d7efd0bfa9485048014a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -125,3 +125,58 @@ xt2ne5el/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text l2vooc6k/episode_rosbags/episode_2_2025-01-04-21-05-23.bag filter=lfs diff=lfs merge=lfs -text cxddxu7z/episode_rosbags/episode_4_2025-01-04-23-02-06.bag filter=lfs diff=lfs merge=lfs -text l2vooc6k/episode_rosbags/episode_0_2025-01-04-21-04-00.bag filter=lfs diff=lfs merge=lfs -text +nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/1_eval_1_f5d85a1429df5e5a3e60.mp4 filter=lfs diff=lfs merge=lfs -text +nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/3_eval_3_c408001aca8abf0c7ef9.mp4 filter=lfs diff=lfs merge=lfs -text +nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/2_eval_2_3966ddb8583e8b4d202c.mp4 filter=lfs diff=lfs merge=lfs -text +nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/0_eval_0_3b79de18ba2122e2eeca.mp4 filter=lfs diff=lfs merge=lfs -text +nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/4_eval_4_d93a45e1282e6c91c0f9.mp4 filter=lfs diff=lfs merge=lfs -text +nhx7r1z0/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nhx7r1z0/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nhx7r1z0/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nhx7r1z0/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nhx7r1z0/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +xt2ne5el/episode_rosbags/episode_1_2025-01-04-22-53-46.bag filter=lfs diff=lfs merge=lfs -text +xt2ne5el/episode_rosbags/episode_2_2025-01-04-22-54-27.bag filter=lfs diff=lfs merge=lfs -text +xt2ne5el/episode_rosbags/episode_3_2025-01-04-22-55-20.bag filter=lfs diff=lfs merge=lfs -text +nhx7r1z0/episode_rosbags/episode_3_2025-01-03-22-32-09.bag filter=lfs diff=lfs merge=lfs -text +nhx7r1z0/episode_rosbags/episode_1_2025-01-03-22-30-21.bag filter=lfs diff=lfs merge=lfs -text +k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/2_eval_2_95cc898a615ff7a378e0.mp4 filter=lfs diff=lfs merge=lfs -text +k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/1_eval_1_92e676c219a8081d0b99.mp4 filter=lfs diff=lfs merge=lfs -text +k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/3_eval_3_ea9f563ddcabcb5060b6.mp4 filter=lfs diff=lfs merge=lfs -text +k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/4_eval_4_c9f7cad01419898a5561.mp4 filter=lfs diff=lfs merge=lfs -text +k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/0_eval_0_242d6ca502a6c74dce29.mp4 filter=lfs diff=lfs merge=lfs -text +k9rmk9dl/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +k9rmk9dl/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +k9rmk9dl/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +k9rmk9dl/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +k9rmk9dl/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nhx7r1z0/episode_rosbags/episode_0_2025-01-03-22-28-51.bag filter=lfs diff=lfs merge=lfs -text +nhx7r1z0/episode_rosbags/episode_2_2025-01-03-22-31-04.bag filter=lfs diff=lfs merge=lfs -text +nhx7r1z0/episode_rosbags/episode_4_2025-01-03-22-32-57.bag filter=lfs diff=lfs merge=lfs -text +k9rmk9dl/episode_rosbags/episode_1_2025-01-03-19-58-48.bag filter=lfs diff=lfs merge=lfs -text +k9rmk9dl/episode_rosbags/episode_3_2025-01-03-20-00-12.bag filter=lfs diff=lfs merge=lfs -text +02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/3_eval_3_4c459657ba9efcfa48fb.mp4 filter=lfs diff=lfs merge=lfs -text +02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/4_eval_4_a3c750bc6c1ba4d47759.mp4 filter=lfs diff=lfs merge=lfs -text +02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/1_eval_1_f60d14b1f2240ddad80d.mp4 filter=lfs diff=lfs merge=lfs -text +02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/0_eval_0_783185f11af79f76b1fa.mp4 filter=lfs diff=lfs merge=lfs -text +02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/2_eval_2_53616642cc0e37a91585.mp4 filter=lfs diff=lfs merge=lfs -text +02thhb4i/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +02thhb4i/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +k9rmk9dl/episode_rosbags/episode_0_2025-01-03-19-58-06.bag filter=lfs diff=lfs merge=lfs -text +02thhb4i/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +02thhb4i/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +02thhb4i/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +k9rmk9dl/episode_rosbags/episode_2_2025-01-03-19-59-30.bag filter=lfs diff=lfs merge=lfs -text +k9rmk9dl/episode_rosbags/episode_4_2025-01-03-20-00-55.bag filter=lfs diff=lfs merge=lfs -text +29yeoadw/episode_rosbags/episode_1_2025-01-04-20-59-40.bag filter=lfs diff=lfs merge=lfs -text +29yeoadw/episode_rosbags/episode_0_2025-01-04-20-59-00.bag filter=lfs diff=lfs merge=lfs -text +qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/0_eval_0_fdd6e363a52ea071e0fd.mp4 filter=lfs diff=lfs merge=lfs -text +qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/2_eval_2_0a6d5b71a8cbdc8ee1f7.mp4 filter=lfs diff=lfs merge=lfs -text +qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/1_eval_1_2c428916b5dd1a4c5cc2.mp4 filter=lfs diff=lfs merge=lfs -text +qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/3_eval_3_5052d6603d25fb3c55cc.mp4 filter=lfs diff=lfs merge=lfs -text +qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/4_eval_4_db949b22d5fdd234cf63.mp4 filter=lfs diff=lfs merge=lfs -text +qulr7kiu/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +qulr7kiu/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +qulr7kiu/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +qulr7kiu/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +qulr7kiu/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/02thhb4i/.hydra/config.yaml b/02thhb4i/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50c4489cc892923b6fde889d218eeedfe242c7ee --- /dev/null +++ b/02thhb4i/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1922_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/02thhb4i/.hydra/hydra.yaml b/02thhb4i/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33904029a31087ab552ec49d084fec4705f5017d --- /dev/null +++ b/02thhb4i/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/02thhb4i/.hydra/overrides.yaml b/02thhb4i/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/02thhb4i/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/02thhb4i/episode_rosbags/aligned_depth_to_color_K.npy b/02thhb4i/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/02thhb4i/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/02thhb4i/episode_rosbags/cam_tf_world.npy b/02thhb4i/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/02thhb4i/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/02thhb4i/episode_rosbags/color_K.npy b/02thhb4i/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/02thhb4i/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/02thhb4i/episode_rosbags/depth_K.npy b/02thhb4i/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/02thhb4i/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/02thhb4i/eval_robot.log b/02thhb4i/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..ab40d610906afce378ddb9897f267b2ca40698fc --- /dev/null +++ b/02thhb4i/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 21:14:12,602][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 21:14:12,605][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/02thhb4i/eval_video/0_eval.mp4 b/02thhb4i/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..89e85f2a9267135d7689878673de423204c4b82a --- /dev/null +++ b/02thhb4i/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:783185f11af79f76b1fa5d5d5579889298440323178d1027d9f4bfd8a72232e6 +size 1155632 diff --git a/02thhb4i/eval_video/1_eval.mp4 b/02thhb4i/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..033438f2d124667e7ffad50a7ba286375761eeec --- /dev/null +++ b/02thhb4i/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f60d14b1f2240ddad80d6204dd549eb0fb0536b64ad1cb94a5357900215e4b1c +size 1164348 diff --git a/02thhb4i/eval_video/2_eval.mp4 b/02thhb4i/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..31bade333eda4c1e1a6015116bcf836b0d5403a8 --- /dev/null +++ b/02thhb4i/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53616642cc0e37a915851b9fa66da62a95025b381fe601321214921114af129e +size 1200672 diff --git a/02thhb4i/eval_video/3_eval.mp4 b/02thhb4i/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fcf18e845a623cd3201cd0b03a9f3fa188e8c728 --- /dev/null +++ b/02thhb4i/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c459657ba9efcfa48fb1256798cfddb4730e411cb8376eb211bab7d60809ef7 +size 1008305 diff --git a/02thhb4i/eval_video/4_eval.mp4 b/02thhb4i/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..340cbc5979162ccd5d6914163ff57ed9d53837e5 --- /dev/null +++ b/02thhb4i/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3c750bc6c1ba4d47759afe590c068752f1d76723ec28d7ffc22cf8237a9a8b0 +size 1164071 diff --git a/02thhb4i/tb/events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 b/02thhb4i/tb/events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 new file mode 100644 index 0000000000000000000000000000000000000000..826b9b2b65a84083ddb353ec12c0c79190aed2db --- /dev/null +++ b/02thhb4i/tb/events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce00520abf23541918f8aba0f47705ef9f5d8f75ab2df7cf59b52a5b14f47e89 +size 6539 diff --git a/02thhb4i/wandb/debug-internal.log b/02thhb4i/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..9c9f772ef0f8359c2191e2aa77819b244e296279 --- /dev/null +++ b/02thhb4i/wandb/debug-internal.log @@ -0,0 +1,725 @@ +2025-01-04 21:14:18,327 INFO StreamThr :2023517 [internal.py:wandb_internal():85] W&B internal server running at pid: 2023517, started at: 2025-01-04 21:14:18.326843 +2025-01-04 21:14:18,328 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status +2025-01-04 21:14:18,342 INFO WriterThread:2023517 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/run-02thhb4i.wandb +2025-01-04 21:14:18,342 DEBUG SenderThread:2023517 [sender.py:send():379] send: header +2025-01-04 21:14:18,349 DEBUG SenderThread:2023517 [sender.py:send():379] send: run +2025-01-04 21:14:18,549 INFO SenderThread:2023517 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files +2025-01-04 21:14:18,549 INFO SenderThread:2023517 [sender.py:_start_run_threads():1188] run started: 02thhb4i with start time 1736043258.326856 +2025-01-04 21:14:18,566 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 21:14:18,567 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: check_version +2025-01-04 21:14:18,612 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 21:14:18,617 DEBUG HandlerThread:2023517 [system_info.py:__init__():26] System info init +2025-01-04 21:14:18,617 DEBUG HandlerThread:2023517 [system_info.py:__init__():41] System info init done +2025-01-04 21:14:18,617 INFO HandlerThread:2023517 [system_monitor.py:start():194] Starting system monitor +2025-01-04 21:14:18,617 INFO SystemMonitor:2023517 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 21:14:18,617 INFO HandlerThread:2023517 [system_monitor.py:probe():214] Collecting system info +2025-01-04 21:14:18,618 INFO SystemMonitor:2023517 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 21:14:18,618 INFO SystemMonitor:2023517 [interfaces.py:start():188] Started disk monitoring +2025-01-04 21:14:18,619 INFO SystemMonitor:2023517 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 21:14:18,620 INFO SystemMonitor:2023517 [interfaces.py:start():188] Started memory monitoring +2025-01-04 21:14:18,620 INFO SystemMonitor:2023517 [interfaces.py:start():188] Started network monitoring +2025-01-04 21:14:18,630 DEBUG HandlerThread:2023517 [system_info.py:probe():152] Probing system +2025-01-04 21:14:18,632 DEBUG HandlerThread:2023517 [system_info.py:_probe_git():137] Probing git +2025-01-04 21:14:18,637 DEBUG HandlerThread:2023517 [system_info.py:_probe_git():145] Probing git done +2025-01-04 21:14:18,637 DEBUG HandlerThread:2023517 [system_info.py:probe():200] Probing system done +2025-01-04 21:14:18,637 DEBUG HandlerThread:2023517 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T02:14:18.630996', 'startedAt': '2025-01-05T02:14:18.313944', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4123.7595, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.71, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.709, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 656.8496742248535}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 21:14:18,637 INFO HandlerThread:2023517 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 21:14:18,637 INFO HandlerThread:2023517 [system_monitor.py:probe():227] Publishing system info +2025-01-04 21:14:18,637 DEBUG HandlerThread:2023517 [system_info.py:_save_code():44] Saving code +2025-01-04 21:14:18,639 DEBUG HandlerThread:2023517 [system_info.py:_save_code():67] Saving code done +2025-01-04 21:14:18,639 DEBUG HandlerThread:2023517 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 21:14:18,701 DEBUG HandlerThread:2023517 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 21:14:18,702 INFO HandlerThread:2023517 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 21:14:18,705 DEBUG SenderThread:2023517 [sender.py:send():379] send: files +2025-01-04 21:14:18,705 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 21:14:18,705 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 21:14:18,706 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 21:14:18,937 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 21:14:18,937 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: python_packages +2025-01-04 21:14:18,938 DEBUG SenderThread:2023517 [sender.py:send():379] send: telemetry +2025-01-04 21:14:18,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:14:18,939 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:14:18,941 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:18,941 INFO HandlerThread:2023517 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/tb" + save: true +} +control { + relay_id: "e8a79328c9e64096bda78ebd56cac1bc" +} +_info { + stream_id: "02thhb4i" +} + +2025-01-04 21:14:18,941 DEBUG HandlerThread:2023517 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 21:14:18,956 INFO wandb-upload_0:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/tix5tzsu-wandb-metadata.json +2025-01-04 21:14:18,983 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/eplsaruh-diff.patch +2025-01-04 21:14:19,012 DEBUG SenderThread:2023517 [sender.py:send():379] send: tbrecord +2025-01-04 21:14:19,012 DEBUG SenderThread:2023517 [sender.py:send():379] send: files +2025-01-04 21:14:19,012 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 with policy live +2025-01-04 21:14:19,034 INFO wandb-upload_1:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/jv1cdi8n-code/FISH/eval_robot.py +2025-01-04 21:14:19,211 INFO wandb-upload_0:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/wvj3b813-events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:14:19,551 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-metadata.json +2025-01-04 21:14:19,551 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/diff.patch +2025-01-04 21:14:19,551 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/requirements.txt +2025-01-04 21:14:19,551 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/code/FISH/eval_robot.py +2025-01-04 21:14:19,551 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:19,552 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:14:19,552 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/code/FISH +2025-01-04 21:14:19,552 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/code +2025-01-04 21:14:19,937 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:20,937 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:21,550 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:21,937 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:22,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:23,341 DEBUG SenderThread:2023517 [sender.py:send():379] send: config +2025-01-04 21:14:23,341 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:14:23,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:24,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:25,551 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:25,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:26,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:27,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:28,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:14:28,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:29,552 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:29,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:30,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:31,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:32,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:33,554 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:33,937 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:14:33,937 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:14:33,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:34,095 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:14:34,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:35,554 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:35,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:36,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:37,555 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:37,940 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:38,940 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:39,555 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:39,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:14:39,940 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:40,942 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:41,941 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:42,941 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:43,942 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:44,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:14:44,942 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:45,942 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:46,942 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:47,942 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:48,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:14:48,938 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:14:48,985 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:49,943 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:50,397 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:14:50,558 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/config.yaml +2025-01-04 21:14:50,943 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:51,558 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:51,943 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:52,943 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:53,943 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:54,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:55,560 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:55,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:56,185 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:14:56,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:57,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:58,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:59,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:00,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:01,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:01,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:02,945 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:03,562 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:03,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:15:03,938 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:15:03,945 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:04,945 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:05,294 DEBUG SenderThread:2023517 [sender.py:send():379] send: files +2025-01-04 21:15:05,294 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_783185f11af79f76b1fa.mp4 with policy now +2025-01-04 21:15:05,294 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:15:05,296 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:15:05,296 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:15:05,297 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:15:05,523 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/jc1mdfnz-media/videos/eval/0_eval_0_783185f11af79f76b1fa.mp4 +2025-01-04 21:15:05,563 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:05,563 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:15:05,563 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/0_eval_0_783185f11af79f76b1fa.mp4 +2025-01-04 21:15:05,563 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media +2025-01-04 21:15:05,563 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval +2025-01-04 21:15:05,563 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos +2025-01-04 21:15:05,828 INFO wandb-upload_0:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/34t5s8my-media/videos/eval/0_eval_0_783185f11af79f76b1fa.mp4 +2025-01-04 21:15:05,945 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:06,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:07,302 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:07,563 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:07,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:08,952 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:09,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:10,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:11,565 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:11,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:12,673 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:12,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:13,565 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:13,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:14,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:15,566 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:15:15,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:16,947 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:17,567 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:17,685 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:17,947 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:18,620 DEBUG SystemMonitor:2023517 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 21:15:18,621 DEBUG SenderThread:2023517 [sender.py:send():379] send: stats +2025-01-04 21:15:18,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:15:18,938 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:15:18,947 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:19,568 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:19,949 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:20,949 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:21,568 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:21,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:22,949 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:23,069 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:23,949 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:24,949 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:25,949 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:26,949 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:27,949 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:28,940 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:28,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:29,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:30,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:31,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:32,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:33,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:15:33,938 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:15:33,981 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:34,106 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:34,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:35,951 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:36,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:37,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:38,952 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:39,943 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:39,951 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:40,951 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:41,951 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:42,951 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:43,951 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:44,945 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:44,952 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:45,952 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:46,399 DEBUG SenderThread:2023517 [sender.py:send():379] send: files +2025-01-04 21:15:46,399 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f60d14b1f2240ddad80d.mp4 with policy now +2025-01-04 21:15:46,402 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:15:46,407 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:15:46,407 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:15:46,408 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:15:46,576 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:15:46,577 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/1_eval_1_f60d14b1f2240ddad80d.mp4 +2025-01-04 21:15:46,577 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval +2025-01-04 21:15:46,664 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/wpvbayt0-media/videos/eval/1_eval_1_f60d14b1f2240ddad80d.mp4 +2025-01-04 21:15:46,932 INFO wandb-upload_0:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/vk852tgt-media/videos/eval/1_eval_1_f60d14b1f2240ddad80d.mp4 +2025-01-04 21:15:46,955 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:47,577 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:47,958 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:48,621 DEBUG SenderThread:2023517 [sender.py:send():379] send: stats +2025-01-04 21:15:48,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:15:48,946 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:15:48,989 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:49,959 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:50,000 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:50,957 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:51,578 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:51,957 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:52,956 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:53,956 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:54,957 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:55,228 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:55,579 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:55,580 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:15:55,812 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/mruyzgn5-events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:15:55,958 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:56,329 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:15:56,330 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:15:56,330 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:15:56,580 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:15:56,957 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:57,580 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:57,957 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:58,957 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:59,957 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:00,483 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:00,957 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:01,581 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:01,958 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:02,965 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:03,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:16:03,939 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:16:03,981 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:04,965 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:05,965 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:06,055 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:06,965 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:07,966 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:08,966 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:09,966 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:10,974 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:11,947 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:11,969 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:12,969 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:13,976 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:14,977 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:15,586 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:15,977 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:16,977 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:17,430 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:17,977 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:18,622 DEBUG SenderThread:2023517 [sender.py:send():379] send: stats +2025-01-04 21:16:18,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:16:18,939 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:16:18,978 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:19,587 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:19,977 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:20,977 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:21,588 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:21,977 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:22,469 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:22,992 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:23,990 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:24,990 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:25,589 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:25,990 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:26,992 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:27,555 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:27,590 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:27,625 DEBUG SenderThread:2023517 [sender.py:send():379] send: files +2025-01-04 21:16:27,626 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_53616642cc0e37a91585.mp4 with policy now +2025-01-04 21:16:27,629 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:16:27,630 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:16:27,630 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:16:27,631 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:16:27,853 INFO wandb-upload_0:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/f06b2bqr-media/videos/eval/2_eval_2_53616642cc0e37a91585.mp4 +2025-01-04 21:16:27,995 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:28,590 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:16:28,591 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/2_eval_2_53616642cc0e37a91585.mp4 +2025-01-04 21:16:28,591 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval +2025-01-04 21:16:28,893 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/zrkhfsep-media/videos/eval/2_eval_2_53616642cc0e37a91585.mp4 +2025-01-04 21:16:28,995 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:29,591 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:29,999 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:31,000 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:32,004 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:32,999 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:33,201 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:33,592 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:33,951 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:16:33,951 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:16:33,997 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:35,008 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:35,593 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:36,006 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:37,001 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:38,001 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:38,951 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:39,008 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:40,007 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:41,002 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:41,162 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:16:41,162 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:16:41,163 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:16:41,595 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:16:41,595 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:16:42,008 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:43,010 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:43,596 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:44,012 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:44,714 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:45,011 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:46,011 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:47,011 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:47,597 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:48,011 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:48,622 DEBUG SenderThread:2023517 [sender.py:send():379] send: stats +2025-01-04 21:16:48,943 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:16:48,943 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:16:49,011 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:50,012 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:50,097 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:51,012 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:52,012 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:53,012 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:54,012 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:55,012 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:55,600 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:55,948 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:56,012 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:57,012 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:58,013 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:59,013 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:00,013 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:00,948 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:01,013 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:02,013 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:03,013 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:03,943 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:17:03,944 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:17:04,013 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:05,013 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:06,013 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:06,043 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:07,024 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:07,604 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:17:07,803 DEBUG SenderThread:2023517 [sender.py:send():379] send: files +2025-01-04 21:17:07,803 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4c459657ba9efcfa48fb.mp4 with policy now +2025-01-04 21:17:07,806 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:17:07,806 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:17:07,806 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:17:07,806 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:17:08,019 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:08,032 INFO wandb-upload_0:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/93ubdiju-media/videos/eval/3_eval_3_4c459657ba9efcfa48fb.mp4 +2025-01-04 21:17:08,604 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:17:08,605 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/3_eval_3_4c459657ba9efcfa48fb.mp4 +2025-01-04 21:17:08,605 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval +2025-01-04 21:17:08,861 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/54j3f85v-media/videos/eval/3_eval_3_4c459657ba9efcfa48fb.mp4 +2025-01-04 21:17:09,024 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:09,605 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:17:10,020 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:11,022 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:11,241 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:12,024 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:13,021 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:13,606 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:17:14,023 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:15,025 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:15,607 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:17:16,028 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:17,024 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:17,069 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:18,030 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:18,623 DEBUG SenderThread:2023517 [sender.py:send():379] send: stats +2025-01-04 21:17:18,949 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:17:18,950 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:17:19,024 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:20,027 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:21,025 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:22,026 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:22,085 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:23,029 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:24,026 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:25,025 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:26,032 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:27,045 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:27,101 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:27,219 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:17:27,219 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:17:27,220 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:17:27,611 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:17:27,611 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:17:27,863 INFO wandb-upload_0:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/eb9z5mk0-events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:17:28,034 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:29,033 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:29,612 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:17:30,031 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:31,029 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:32,029 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:32,360 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:33,029 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:33,613 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:17:33,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:17:33,944 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:17:34,034 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:35,034 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:36,034 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:37,035 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:38,016 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:38,035 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:39,038 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:40,038 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:41,039 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:42,039 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:43,039 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:43,952 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:44,040 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:45,040 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:46,040 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:47,040 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:48,040 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:48,623 DEBUG SenderThread:2023517 [sender.py:send():379] send: stats +2025-01-04 21:17:48,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:17:48,944 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:17:49,040 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:49,063 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:50,041 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:51,041 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:52,041 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:53,041 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:54,041 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:54,953 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:55,041 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:55,621 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:17:56,041 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:57,041 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:58,042 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:59,051 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:59,373 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:17:59,373 DEBUG SenderThread:2023517 [sender.py:send():379] send: files +2025-01-04 21:17:59,373 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_a3c750bc6c1ba4d47759.mp4 with policy now +2025-01-04 21:17:59,374 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:17:59,374 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:17:59,374 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:17:59,622 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:17:59,623 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:17:59,623 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/4_eval_4_a3c750bc6c1ba4d47759.mp4 +2025-01-04 21:17:59,623 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval +2025-01-04 21:17:59,627 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/3ms2sgi5-media/videos/eval/4_eval_4_a3c750bc6c1ba4d47759.mp4 +2025-01-04 21:17:59,881 INFO wandb-upload_0:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/2k81dwu3-media/videos/eval/4_eval_4_a3c750bc6c1ba4d47759.mp4 +2025-01-04 21:18:00,048 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:00,420 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:18:01,054 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:01,623 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:18:02,052 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:03,077 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:03,952 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:18:03,952 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:18:04,059 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:05,064 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:05,624 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:18:05,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:18:06,072 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:07,057 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:07,625 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:18:08,055 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:09,063 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:10,070 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:10,272 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:18:10,273 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:10,273 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:10,338 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:18:10,338 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:18:10,524 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:18:10,626 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:18:10,626 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:18:10,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-04 21:18:10,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media +2025-01-04 21:18:10,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table/eval +2025-01-04 21:18:10,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table +2025-01-04 21:18:10,803 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/7hvggdtl-media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-04 21:18:10,877 INFO wandb-upload_0:2023517 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwadsqzmy +2025-01-04 21:18:10,952 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:18:10,962 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:18:11,059 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:11,302 INFO SenderThread:2023517 [sender.py:send_request_log_artifact():1525] logged artifact run-02thhb4i-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE0MzgzOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1NDE3Mg==', 'latestArtifact': None}} +2025-01-04 21:18:11,302 DEBUG SenderThread:2023517 [sender.py:send():379] send: files +2025-01-04 21:18:11,302 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json with policy now +2025-01-04 21:18:11,303 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:18:11,303 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:18:11,303 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,304 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,304 DEBUG SenderThread:2023517 [sender.py:send():379] send: telemetry +2025-01-04 21:18:11,304 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,308 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,308 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,308 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,308 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,309 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,309 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,309 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,309 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,310 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,310 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,310 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,310 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,311 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,311 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,311 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,311 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,312 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,312 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,312 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,312 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,313 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,313 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:18:11,537 INFO wandb-upload_1:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/48xcex5h-media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-04 21:18:11,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:18:11,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:18:11,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:18:11,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table/eval/total_success_rate_ci_6_4fb6272d24a89dcc8336.table.json +2025-01-04 21:18:11,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table/eval +2025-01-04 21:18:11,652 INFO wandb-upload_0:2023517 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpj9uhk76r +2025-01-04 21:18:11,812 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/zklxcxi0-media/table/eval/total_success_rate_ci_6_4fb6272d24a89dcc8336.table.json +2025-01-04 21:18:12,054 INFO SenderThread:2023517 [sender.py:send_request_log_artifact():1525] logged artifact run-02thhb4i-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE0Mzg1MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1NDE3NA==', 'latestArtifact': None}} +2025-01-04 21:18:12,054 DEBUG SenderThread:2023517 [sender.py:send():379] send: files +2025-01-04 21:18:12,054 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_4fb6272d24a89dcc8336.table.json with policy now +2025-01-04 21:18:12,055 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:18:12,055 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:12,055 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:12,059 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:12,278 INFO wandb-upload_1:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/pws6kvzy-media/table/eval/total_success_rate_ci_6_4fb6272d24a89dcc8336.table.json +2025-01-04 21:18:12,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:18:13,059 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:13,558 DEBUG SenderThread:2023517 [sender.py:send():379] send: exit +2025-01-04 21:18:13,558 INFO SenderThread:2023517 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 21:18:13,558 INFO SenderThread:2023517 [sender.py:send_exit():588] handling runtime: 234 +2025-01-04 21:18:13,558 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:13,558 INFO SenderThread:2023517 [sender.py:send_exit():594] send defer +2025-01-04 21:18:13,558 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:13,558 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 21:18:13,559 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:13,559 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 21:18:13,559 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 1 +2025-01-04 21:18:13,559 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:13,559 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 21:18:13,559 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:13,559 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 21:18:13,559 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 2 +2025-01-04 21:18:13,559 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:13,559 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 21:18:13,559 INFO HandlerThread:2023517 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 21:18:13,559 DEBUG SystemMonitor:2023517 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 21:18:13,559 DEBUG SystemMonitor:2023517 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 21:18:13,560 INFO HandlerThread:2023517 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 21:18:13,560 INFO HandlerThread:2023517 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 21:18:13,614 INFO HandlerThread:2023517 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 21:18:13,615 INFO HandlerThread:2023517 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 21:18:13,615 INFO HandlerThread:2023517 [interfaces.py:finish():200] Joined network monitor +2025-01-04 21:18:13,615 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:13,615 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 21:18:13,615 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 3 +2025-01-04 21:18:13,615 DEBUG SenderThread:2023517 [sender.py:send():379] send: stats +2025-01-04 21:18:13,615 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:13,616 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 21:18:13,616 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:13,616 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 21:18:13,616 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 4 +2025-01-04 21:18:13,616 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:13,616 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 21:18:13,628 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:18:13,628 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:18:15,628 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:18:20,278 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:18:20,278 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:18:20,278 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 21:18:20,280 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:20,280 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 21:18:20,280 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 5 +2025-01-04 21:18:20,280 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:18:20,280 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:18:20,280 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:20,280 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:20,280 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 21:18:20,281 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:20,281 DEBUG SenderThread:2023517 [sender.py:send():379] send: summary +2025-01-04 21:18:20,281 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:20,282 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:20,282 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 21:18:20,282 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 6 +2025-01-04 21:18:20,282 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:20,282 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 21:18:20,282 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:20,282 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 21:18:20,285 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:18:20,407 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 7 +2025-01-04 21:18:20,407 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:20,407 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 21:18:20,407 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:20,407 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 21:18:20,559 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:18:20,631 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/config.yaml +2025-01-04 21:18:20,631 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:18:21,668 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 8 +2025-01-04 21:18:21,668 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:18:21,669 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:21,669 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 21:18:21,669 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:21,669 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 21:18:21,669 INFO SenderThread:2023517 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 21:18:21,671 INFO SenderThread:2023517 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 21:18:21,673 INFO SenderThread:2023517 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 21:18:21,682 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 9 +2025-01-04 21:18:21,682 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:21,682 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 21:18:21,683 DEBUG SenderThread:2023517 [sender.py:send():379] send: artifact +2025-01-04 21:18:21,893 INFO SenderThread:2023517 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 21:18:21,893 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:21,893 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 21:18:21,893 INFO SenderThread:2023517 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 21:18:22,559 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:18:22,632 INFO SenderThread:2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:18:22,632 INFO SenderThread:2023517 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files +2025-01-04 21:18:22,632 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/config.yaml config.yaml +2025-01-04 21:18:22,632 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json wandb-summary.json +2025-01-04 21:18:22,632 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/requirements.txt requirements.txt +2025-01-04 21:18:22,632 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-metadata.json wandb-metadata.json +2025-01-04 21:18:22,632 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log output.log +2025-01-04 21:18:22,633 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/diff.patch diff.patch +2025-01-04 21:18:22,633 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:18:22,640 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 21:18:22,641 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/0_eval_0_783185f11af79f76b1fa.mp4 media/videos/eval/0_eval_0_783185f11af79f76b1fa.mp4 +2025-01-04 21:18:22,641 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/3_eval_3_4c459657ba9efcfa48fb.mp4 media/videos/eval/3_eval_3_4c459657ba9efcfa48fb.mp4 +2025-01-04 21:18:22,641 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/2_eval_2_53616642cc0e37a91585.mp4 media/videos/eval/2_eval_2_53616642cc0e37a91585.mp4 +2025-01-04 21:18:22,641 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/1_eval_1_f60d14b1f2240ddad80d.mp4 media/videos/eval/1_eval_1_f60d14b1f2240ddad80d.mp4 +2025-01-04 21:18:22,641 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/4_eval_4_a3c750bc6c1ba4d47759.mp4 media/videos/eval/4_eval_4_a3c750bc6c1ba4d47759.mp4 +2025-01-04 21:18:22,641 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table/eval/total_success_rate_ci_6_4fb6272d24a89dcc8336.table.json media/table/eval/total_success_rate_ci_6_4fb6272d24a89dcc8336.table.json +2025-01-04 21:18:22,642 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-04 21:18:22,642 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 10 +2025-01-04 21:18:22,642 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:18:22,643 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:22,643 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 21:18:22,644 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:22,644 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 21:18:22,644 INFO SenderThread:2023517 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:18:22,839 INFO wandb-upload_3:2023517 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/config.yaml +2025-01-04 21:18:22,844 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:18:22,858 INFO wandb-upload_0:2023517 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/requirements.txt +2025-01-04 21:18:22,912 INFO wandb-upload_1:2023517 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:18:22,948 INFO wandb-upload_4:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/wdg0y3tm-events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:18:23,148 INFO Thread-11 (_thread_body):2023517 [sender.py:transition_state():614] send defer: 11 +2025-01-04 21:18:23,149 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:23,149 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 21:18:23,149 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:23,149 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 21:18:23,149 INFO SenderThread:2023517 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 21:18:23,150 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 12 +2025-01-04 21:18:23,150 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:23,150 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 21:18:23,150 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:23,151 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 21:18:23,151 INFO SenderThread:2023517 [file_stream.py:finish():601] file stream finish called +2025-01-04 21:18:23,528 INFO SenderThread:2023517 [file_stream.py:finish():605] file stream finish is done +2025-01-04 21:18:23,528 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 13 +2025-01-04 21:18:23,528 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:23,528 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 21:18:23,528 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:23,529 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 21:18:23,529 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 14 +2025-01-04 21:18:23,529 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:23,529 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 21:18:23,529 DEBUG SenderThread:2023517 [sender.py:send():379] send: final +2025-01-04 21:18:23,529 DEBUG SenderThread:2023517 [sender.py:send():379] send: footer +2025-01-04 21:18:23,529 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:23,529 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 21:18:23,530 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:18:23,530 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:18:23,530 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 21:18:23,530 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: server_info +2025-01-04 21:18:23,532 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:18:23,532 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 21:18:23,532 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 21:18:23,533 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:23,591 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:18:23,591 INFO MainThread:2023517 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 21:18:23,592 INFO MainThread:2023517 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 21:18:23,592 INFO MainThread:2023517 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 21:18:23,592 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 21:18:23,592 INFO HandlerThread:2023517 [handler.py:finish():882] shutting down handler +2025-01-04 21:18:24,532 INFO WriterThread:2023517 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/run-02thhb4i.wandb +2025-01-04 21:18:24,591 INFO SenderThread:2023517 [sender.py:finish():1615] shutting down sender +2025-01-04 21:18:24,591 INFO SenderThread:2023517 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:18:24,591 INFO SenderThread:2023517 [file_pusher.py:join():175] waiting for file pusher diff --git a/02thhb4i/wandb/debug.log b/02thhb4i/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7af320536742dd90dd9b01efc89b590ccfd1cbdf --- /dev/null +++ b/02thhb4i/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 21:14:18,320 INFO MainThread:2023350 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_setup.py:_flush():76] Configure stats pid to 2023350 +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/settings +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/logs/debug.log +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/logs/debug-internal.log +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_init.py:init():569] calling init triggers +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i'} +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_init.py:init():619] starting backend +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_init.py:init():623] setting up manager +2025-01-04 21:14:18,326 INFO MainThread:2023350 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 21:14:18,326 INFO MainThread:2023350 [wandb_init.py:init():631] backend started and connected +2025-01-04 21:14:18,340 INFO MainThread:2023350 [wandb_init.py:init():720] updated telemetry +2025-01-04 21:14:18,347 INFO MainThread:2023350 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 21:14:18,566 INFO MainThread:2023350 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 21:14:18,607 INFO MainThread:2023350 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 21:14:18,607 INFO MainThread:2023350 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 21:14:18,937 INFO MainThread:2023350 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 21:14:18,937 INFO MainThread:2023350 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 21:14:18,937 INFO MainThread:2023350 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 21:14:18,937 INFO MainThread:2023350 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 21:14:18,939 INFO MainThread:2023350 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 21:14:18,939 INFO MainThread:2023350 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/tb, True +2025-01-04 21:14:23,340 INFO MainThread:2023350 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-04 21:18:24,593 WARNING MsgRouterThr:2023350 [router.py:message_loop():77] message_loop has been closed diff --git a/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/code/FISH/eval_robot.py b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/config.yaml b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad4eb485fdf143f7ba503ff4a00dbd4ca6a8cc22 --- /dev/null +++ b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1922_1' +wandb_notes: + desc: null + value: 1922_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736043258 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/diff.patch b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b8501904dd167f150d5cfd5100b04efef6d61ed1 --- /dev/null +++ b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..d2afe29 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json new file mode 100644 index 0000000000000000000000000000000000000000..a2db2594aac2dfee94d7430f5fc4332216ce7fe1 --- /dev/null +++ b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.8, 0.37485200169205224, 0.9957208297155283, 0.3695220351480134, 0.9926368444172731, 0.4414144158480513, 0.954582236616294, 0.4180309623643781, 0.934476241382488]]} \ No newline at end of file diff --git a/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table/eval/total_success_rate_ci_6_4fb6272d24a89dcc8336.table.json b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table/eval/total_success_rate_ci_6_4fb6272d24a89dcc8336.table.json new file mode 100644 index 0000000000000000000000000000000000000000..ab11d9c78fa13f7fca1fbb17ace818b8a8f6919f --- /dev/null +++ b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table/eval/total_success_rate_ci_6_4fb6272d24a89dcc8336.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[42, 50, 0.84, 0.7076379805388022, 0.9311527161927993, 0.7140152602959804, 0.9183711855766881, 0.7316220663024668, 0.9124258457789498, 0.7732925563504089, 0.9007318726870637]]} \ No newline at end of file diff --git a/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/0_eval_0_783185f11af79f76b1fa.mp4 b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/0_eval_0_783185f11af79f76b1fa.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..89e85f2a9267135d7689878673de423204c4b82a --- /dev/null +++ b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/0_eval_0_783185f11af79f76b1fa.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:783185f11af79f76b1fa5d5d5579889298440323178d1027d9f4bfd8a72232e6 +size 1155632 diff --git a/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/1_eval_1_f60d14b1f2240ddad80d.mp4 b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/1_eval_1_f60d14b1f2240ddad80d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..033438f2d124667e7ffad50a7ba286375761eeec --- /dev/null +++ b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/1_eval_1_f60d14b1f2240ddad80d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f60d14b1f2240ddad80d6204dd549eb0fb0536b64ad1cb94a5357900215e4b1c +size 1164348 diff --git a/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/2_eval_2_53616642cc0e37a91585.mp4 b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/2_eval_2_53616642cc0e37a91585.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..31bade333eda4c1e1a6015116bcf836b0d5403a8 --- /dev/null +++ b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/2_eval_2_53616642cc0e37a91585.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53616642cc0e37a915851b9fa66da62a95025b381fe601321214921114af129e +size 1200672 diff --git a/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/3_eval_3_4c459657ba9efcfa48fb.mp4 b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/3_eval_3_4c459657ba9efcfa48fb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fcf18e845a623cd3201cd0b03a9f3fa188e8c728 --- /dev/null +++ b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/3_eval_3_4c459657ba9efcfa48fb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c459657ba9efcfa48fb1256798cfddb4730e411cb8376eb211bab7d60809ef7 +size 1008305 diff --git a/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/4_eval_4_a3c750bc6c1ba4d47759.mp4 b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/4_eval_4_a3c750bc6c1ba4d47759.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..340cbc5979162ccd5d6914163ff57ed9d53837e5 --- /dev/null +++ b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/4_eval_4_a3c750bc6c1ba4d47759.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3c750bc6c1ba4d47759afe590c068752f1d76723ec28d7ffc22cf8237a9a8b0 +size 1164071 diff --git a/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c52218abfee3f8becb339ffb59f05d682e36650a --- /dev/null +++ b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log @@ -0,0 +1,284 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736043263.341780]: resetting environment +[INFO] [1736043263.346689]: cleared current plan +[INFO] [1736043263.347924]: moving to home +[INFO] [1736043264.350688]: reached home +[INFO] [1736043264.352379]: reset action history +[INFO] [1736043267.061198]: environment reset +Starting episode 0 +[INFO] [1736043267.064396]: resetting environment +[INFO] [1736043267.069564]: cleared current plan +[INFO] [1736043267.071240]: moving to home +[INFO] [1736043268.074386]: reached home +[INFO] [1736043268.076075]: reset action history +[INFO] [1736043270.784150]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736043273.597103]: resetting environment +[INFO] [1736043273.600175]: cleared current plan +[INFO] [1736043273.601104]: moving to home +[INFO] [1736043274.603545]: reached home +[INFO] [1736043274.604720]: reset action history +[INFO] [1736043277.313722]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/episode_rosbags/episode_0_2025-01-04-21-14-37.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2023350.log +started roslaunch server http://158.130.50.37:34603/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2023615] +process[print_text-2]: started with pid [2023616] +process[rosbag_record-3]: started with pid [2023640] +started bagging! +For topic gripper_width: timestamp difference is -113652997 for nearest: 1736043289151580758 - target: 1736043289265233755 at idx 166 +For topic gripper_width: timestamp difference is -103781162 for nearest: 1736043293951561274 - target: 1736043294055342436 at idx 236 +For topic gripper_width: timestamp difference is 114214760 for nearest: 1736043302951555592 - target: 1736043302837340832 at idx 249 +[INFO] [1736043304.314649]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736043305.087154]: Stored episode 1. +[INFO] [1736043305.088238]: Saving video... +[INFO] [1736043305.283556]: Video saved! +Starting episode 1 +[INFO] [1736043305.295804]: resetting environment +[INFO] [1736043305.301889]: cleared current plan +[INFO] [1736043305.302195]: moving to home +[INFO] [1736043308.904609]: reached home +[INFO] [1736043308.911988]: reset action history +[INFO] [1736043311.625979]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736043314.640871]: resetting environment +[INFO] [1736043314.643132]: cleared current plan +[INFO] [1736043314.643281]: moving to home +[INFO] [1736043315.644948]: reached home +[INFO] [1736043315.645362]: reset action history +[INFO] [1736043318.353033]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/episode_rosbags/episode_1_2025-01-04-21-15-18.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2023350.log +started roslaunch server http://158.130.50.37:36785/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2023750] +process[print_text-5]: started with pid [2023751] +process[rosbag_record-6]: started with pid [2023775] +started bagging! +[INFO] [1736043345.382753]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736043346.140115]: Stored episode 2. +[INFO] [1736043346.140909]: Saving video... +[INFO] [1736043346.368094]: Video saved! +Starting episode 2 +[INFO] [1736043346.407633]: resetting environment +[INFO] [1736043346.428598]: cleared current plan +[INFO] [1736043346.432189]: moving to home +[INFO] [1736043350.050702]: reached home +[INFO] [1736043350.055729]: reset action history +[INFO] [1736043352.824454]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736043355.249712]: resetting environment +[INFO] [1736043355.259199]: cleared current plan +[INFO] [1736043355.259618]: moving to home +[INFO] [1736043356.262807]: reached home +[INFO] [1736043356.263391]: reset action history +[INFO] [1736043358.970906]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/episode_rosbags/episode_2_2025-01-04-21-15-58.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2023350.log +started roslaunch server http://158.130.50.37:33577/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2023909] +process[print_text-8]: started with pid [2023910] +process[rosbag_record-9]: started with pid [2023911] +started bagging! +For topic gripper_width: timestamp difference is 100018972 for nearest: 1736043373418247932 - target: 1736043373318228960 at idx 203 +For topic gripper_width: timestamp difference is 100605799 for nearest: 1736043374218190028 - target: 1736043374117584229 at idx 214 +For topic gripper_width: timestamp difference is 100759685 for nearest: 1736043374418183267 - target: 1736043374317423582 at idx 216 +For topic gripper_width: timestamp difference is 103449829 for nearest: 1736043377618217953 - target: 1736043377514768124 at idx 249 +For topic gripper_width: timestamp difference is 103603357 for nearest: 1736043377818222278 - target: 1736043377714618921 at idx 249 +For topic gripper_width: timestamp difference is 104196948 for nearest: 1736043378618222636 - target: 1736043378514025688 at idx 249 +For topic gripper_width: timestamp difference is 104791578 for nearest: 1736043379418215450 - target: 1736043379313423872 at idx 249 +For topic gripper_width: timestamp difference is 108620340 for nearest: 1736043384218224699 - target: 1736043384109604359 at idx 249 +[INFO] [1736043386.210026]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1736043386.963263]: Stored episode 3. +[INFO] [1736043386.964845]: Saving video... +[INFO] [1736043387.600407]: Video saved! +Starting episode 3 +[INFO] [1736043387.633093]: resetting environment +[INFO] [1736043387.646736]: cleared current plan +[INFO] [1736043387.651503]: moving to home +[WARN] [1736043387.551750]: Plan exhausted +[WARN] [1736043387.590995]: Plan exhausted +[WARN] [1736043387.634901]: Plan exhausted +[INFO] [1736043391.157565]: reached home +[INFO] [1736043391.165114]: reset action history +[INFO] [1736043393.954718]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736043400.803465]: resetting environment +[INFO] [1736043400.815657]: cleared current plan +[INFO] [1736043400.815862]: moving to home +[INFO] [1736043401.817530]: reached home +[INFO] [1736043401.824479]: reset action history +For topic color: timestamp difference is -133227348 for nearest: 1736043404526353598 - target: 1736043404659580946 at idx 29 +[INFO] [1736043404.744295]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/episode_rosbags/episode_3_2025-01-04-21-16-44.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2023350.log +started roslaunch server http://158.130.50.37:45855/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2024024] +process[print_text-11]: started with pid [2024025] +process[rosbag_record-12]: started with pid [2024049] +started bagging! +For topic gripper_width: timestamp difference is -100739272 for nearest: 1736043413551550834 - target: 1736043413652290106 at idx 127 +preempting episode +[INFO] [1736043426.421803]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736043427.225070]: Stored episode 4. +[INFO] [1736043427.228822]: Saving video... +[INFO] [1736043427.767954]: Video saved! +Starting episode 4 +[INFO] [1736043427.810214]: resetting environment +[INFO] [1736043427.825974]: cleared current plan +[INFO] [1736043427.827596]: moving to home +[WARN] [1736043427.711972]: Plan exhausted +[WARN] [1736043427.754702]: Plan exhausted +[WARN] [1736043427.790975]: Plan exhausted +[INFO] [1736043431.237949]: reached home +[INFO] [1736043431.241053]: reset action history +[INFO] [1736043434.024513]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736043447.040617]: resetting environment +[INFO] [1736043447.054646]: cleared current plan +[INFO] [1736043447.056525]: moving to home +[INFO] [1736043448.059918]: reached home +[INFO] [1736043448.070789]: reset action history +[INFO] [1736043450.862228]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/episode_rosbags/episode_4_2025-01-04-21-17-30.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2023350.log +started roslaunch server http://158.130.50.37:32779/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2024183] +process[print_text-14]: started with pid [2024184] +process[rosbag_record-15]: started with pid [2024185] +started bagging! +For topic gripper_width: timestamp difference is -252298087 for nearest: 1736043473351551801 - target: 1736043473603849888 at idx 249 +[INFO] [1736043478.065439]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736043478.869350]: Stored episode 5. +[INFO] [1736043478.882053]: Saving video... +[INFO] [1736043479.324710]: Video saved! +[INFO] [1736043479.373686]: resetting environment +[INFO] [1736043479.403840]: cleared current plan +[INFO] [1736043479.418928]: moving to home +[INFO] [1736043483.027728]: reached home +[INFO] [1736043483.035764]: reset action history +[INFO] [1736043485.912288]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/requirements.txt b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-metadata.json b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c156dc00c88d05e0a59218e445c29a35d531acd0 --- /dev/null +++ b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T02:14:18.630996", + "startedAt": "2025-01-05T02:14:18.313944", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4123.7595, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.71, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4498.709, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 656.8496742248535 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c7e35daaaeaf62cfa24b3fb90b6ad3e7f10e67a6 --- /dev/null +++ b/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "783185f11af79f76b1fa5d5d5579889298440323178d1027d9f4bfd8a72232e6", "size": 1155632, "path": "media/videos/eval/0_eval_0_783185f11af79f76b1fa.mp4"}, "global_step": 638, "_timestamp": 1736043490.8557262, "_runtime": 232.528870344162, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "f60d14b1f2240ddad80d6204dd549eb0fb0536b64ad1cb94a5357900215e4b1c", "size": 1164348, "path": "media/videos/eval/1_eval_1_f60d14b1f2240ddad80d.mp4"}, "eval/num_success": 4.0, "episode": 50.0, "eval/success_rate": 0.800000011920929, "eval/success_rate_lb_0.01": 0.3748520016670227, "eval/success_rate_ub_0.01": 0.9957208037376404, "eval/success_rate_lb_0.025": 0.3695220351219177, "eval/success_rate_ub_0.025": 0.9926368594169617, "eval/success_rate_lb_0.05": 0.44141441583633423, "eval/success_rate_ub_0.05": 0.9545822143554688, "eval/success_rate_lb_0.1": 0.4180309772491455, "eval/success_rate_ub_0.1": 0.9344762563705444, "eval/2_eval": {"_type": "video-file", "sha256": "53616642cc0e37a915851b9fa66da62a95025b381fe601321214921114af129e", "size": 1200672, "path": "media/videos/eval/2_eval_2_53616642cc0e37a91585.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "4c459657ba9efcfa48fb1256798cfddb4730e411cb8376eb211bab7d60809ef7", "size": 1008305, "path": "media/videos/eval/3_eval_3_4c459657ba9efcfa48fb.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "a3c750bc6c1ba4d47759afe590c068752f1d76723ec28d7ffc22cf8237a9a8b0", "size": 1164071, "path": "media/videos/eval/4_eval_4_a3c750bc6c1ba4d47759.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "8490adedbf62c026e30853c69820c3dd6a1c7da4ee481eab9e8dcf125e285d47", "size": 397, "artifact_path": "wandb-client-artifact://54myf5oaiqwgmb5rlxay4j76cxrwb5x2mtykpp1ch5q41pkpuhzr672glbxq5ivfq8vopg3mfz5hifdx9pxr2dx2hsr7ww07iusvyrh71ppcwyenij2imt61fqmnymva/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://9gt67delbxwbbfiar7lpddqml07xu48itdj5u1sbfit28ea0tvem3gjkmi9b5zvoml8hk8qzet10rgtxbsfkvjwd6tfhf1svjrodt1sji8w5ub6wvl679lce0t2jbglh:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 42, "total_num_episodes": 50, "total_success_rate": 0.84, "total_success_rate_lb_0.01": 0.7076379805388022, "total_success_rate_ub_0.01": 0.9311527161927993, "total_success_rate_lb_0.025": 0.7140152602959804, "total_success_rate_ub_0.025": 0.9183711855766881, "total_success_rate_lb_0.05": 0.7316220663024668, "total_success_rate_ub_0.05": 0.9124258457789498, "total_success_rate_lb_0.1": 0.7732925563504089, "total_success_rate_ub_0.1": 0.9007318726870637, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "4fb6272d24a89dcc8336d3171713d9125d3f80284da74d35409c152777988ad9", "size": 506, "artifact_path": "wandb-client-artifact://nbrvhmyivqoltbgj82tipjxn9aa8jvd2va4oj71g9x596z5fgi8ytg3rvsg3150e3l5lvjjcvhmjecaz1wuk5k2io94a0r9s62xwh3nes5k23iwha899g8kzbilt7182/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://80vrrw37hoj9va9xjb9l1fd4xwk391w1u0v7y1kzwo7t4xtqjz26twmrgqjw69ep4ba74d7tvrrxhdh5bok6l04qsjnytxnnbo7okdd9kde6vzfl4n6jn4w2sb0986hu:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_4fb6272d24a89dcc8336.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 234}, "eval/total_success_rate": 0.8399999737739563, "eval/total_success_rate_lb_0.01": 0.7076379656791687, "eval/total_success_rate_ub_0.01": 0.9311527013778687, "eval/total_success_rate_lb_0.025": 0.7140152454376221, "eval/total_success_rate_ub_0.025": 0.9183712005615234, "eval/total_success_rate_lb_0.05": 0.731622040271759, "eval/total_success_rate_ub_0.05": 0.9124258160591125, "eval/total_success_rate_lb_0.1": 0.7732925415039062, "eval/total_success_rate_ub_0.1": 0.9007318615913391} \ No newline at end of file diff --git a/02thhb4i/wandb/run-20250104_211418-02thhb4i/logs/debug-internal.log b/02thhb4i/wandb/run-20250104_211418-02thhb4i/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..9c9f772ef0f8359c2191e2aa77819b244e296279 --- /dev/null +++ b/02thhb4i/wandb/run-20250104_211418-02thhb4i/logs/debug-internal.log @@ -0,0 +1,725 @@ +2025-01-04 21:14:18,327 INFO StreamThr :2023517 [internal.py:wandb_internal():85] W&B internal server running at pid: 2023517, started at: 2025-01-04 21:14:18.326843 +2025-01-04 21:14:18,328 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status +2025-01-04 21:14:18,342 INFO WriterThread:2023517 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/run-02thhb4i.wandb +2025-01-04 21:14:18,342 DEBUG SenderThread:2023517 [sender.py:send():379] send: header +2025-01-04 21:14:18,349 DEBUG SenderThread:2023517 [sender.py:send():379] send: run +2025-01-04 21:14:18,549 INFO SenderThread:2023517 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files +2025-01-04 21:14:18,549 INFO SenderThread:2023517 [sender.py:_start_run_threads():1188] run started: 02thhb4i with start time 1736043258.326856 +2025-01-04 21:14:18,566 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 21:14:18,567 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: check_version +2025-01-04 21:14:18,612 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 21:14:18,617 DEBUG HandlerThread:2023517 [system_info.py:__init__():26] System info init +2025-01-04 21:14:18,617 DEBUG HandlerThread:2023517 [system_info.py:__init__():41] System info init done +2025-01-04 21:14:18,617 INFO HandlerThread:2023517 [system_monitor.py:start():194] Starting system monitor +2025-01-04 21:14:18,617 INFO SystemMonitor:2023517 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 21:14:18,617 INFO HandlerThread:2023517 [system_monitor.py:probe():214] Collecting system info +2025-01-04 21:14:18,618 INFO SystemMonitor:2023517 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 21:14:18,618 INFO SystemMonitor:2023517 [interfaces.py:start():188] Started disk monitoring +2025-01-04 21:14:18,619 INFO SystemMonitor:2023517 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 21:14:18,620 INFO SystemMonitor:2023517 [interfaces.py:start():188] Started memory monitoring +2025-01-04 21:14:18,620 INFO SystemMonitor:2023517 [interfaces.py:start():188] Started network monitoring +2025-01-04 21:14:18,630 DEBUG HandlerThread:2023517 [system_info.py:probe():152] Probing system +2025-01-04 21:14:18,632 DEBUG HandlerThread:2023517 [system_info.py:_probe_git():137] Probing git +2025-01-04 21:14:18,637 DEBUG HandlerThread:2023517 [system_info.py:_probe_git():145] Probing git done +2025-01-04 21:14:18,637 DEBUG HandlerThread:2023517 [system_info.py:probe():200] Probing system done +2025-01-04 21:14:18,637 DEBUG HandlerThread:2023517 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T02:14:18.630996', 'startedAt': '2025-01-05T02:14:18.313944', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4123.7595, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.71, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.709, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 656.8496742248535}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 21:14:18,637 INFO HandlerThread:2023517 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 21:14:18,637 INFO HandlerThread:2023517 [system_monitor.py:probe():227] Publishing system info +2025-01-04 21:14:18,637 DEBUG HandlerThread:2023517 [system_info.py:_save_code():44] Saving code +2025-01-04 21:14:18,639 DEBUG HandlerThread:2023517 [system_info.py:_save_code():67] Saving code done +2025-01-04 21:14:18,639 DEBUG HandlerThread:2023517 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 21:14:18,701 DEBUG HandlerThread:2023517 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 21:14:18,702 INFO HandlerThread:2023517 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 21:14:18,705 DEBUG SenderThread:2023517 [sender.py:send():379] send: files +2025-01-04 21:14:18,705 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 21:14:18,705 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 21:14:18,706 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 21:14:18,937 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 21:14:18,937 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: python_packages +2025-01-04 21:14:18,938 DEBUG SenderThread:2023517 [sender.py:send():379] send: telemetry +2025-01-04 21:14:18,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:14:18,939 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:14:18,941 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:18,941 INFO HandlerThread:2023517 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/tb" + save: true +} +control { + relay_id: "e8a79328c9e64096bda78ebd56cac1bc" +} +_info { + stream_id: "02thhb4i" +} + +2025-01-04 21:14:18,941 DEBUG HandlerThread:2023517 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 21:14:18,956 INFO wandb-upload_0:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/tix5tzsu-wandb-metadata.json +2025-01-04 21:14:18,983 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/eplsaruh-diff.patch +2025-01-04 21:14:19,012 DEBUG SenderThread:2023517 [sender.py:send():379] send: tbrecord +2025-01-04 21:14:19,012 DEBUG SenderThread:2023517 [sender.py:send():379] send: files +2025-01-04 21:14:19,012 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 with policy live +2025-01-04 21:14:19,034 INFO wandb-upload_1:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/jv1cdi8n-code/FISH/eval_robot.py +2025-01-04 21:14:19,211 INFO wandb-upload_0:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/wvj3b813-events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:14:19,551 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-metadata.json +2025-01-04 21:14:19,551 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/diff.patch +2025-01-04 21:14:19,551 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/requirements.txt +2025-01-04 21:14:19,551 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/code/FISH/eval_robot.py +2025-01-04 21:14:19,551 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:19,552 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:14:19,552 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/code/FISH +2025-01-04 21:14:19,552 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/code +2025-01-04 21:14:19,937 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:20,937 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:21,550 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:21,937 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:22,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:23,341 DEBUG SenderThread:2023517 [sender.py:send():379] send: config +2025-01-04 21:14:23,341 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:14:23,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:24,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:25,551 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:25,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:26,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:27,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:28,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:14:28,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:29,552 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:29,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:30,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:31,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:32,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:33,554 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:33,937 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:14:33,937 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:14:33,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:34,095 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:14:34,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:35,554 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:35,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:36,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:37,555 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:37,940 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:38,940 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:39,555 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:39,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:14:39,940 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:40,942 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:41,941 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:42,941 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:43,942 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:44,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:14:44,942 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:45,942 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:46,942 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:47,942 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:48,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:14:48,938 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:14:48,985 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:49,943 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:50,397 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:14:50,558 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/config.yaml +2025-01-04 21:14:50,943 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:51,558 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:51,943 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:52,943 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:53,943 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:54,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:55,560 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:14:55,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:56,185 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:14:56,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:57,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:58,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:14:59,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:00,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:01,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:01,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:02,945 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:03,562 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:03,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:15:03,938 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:15:03,945 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:04,945 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:05,294 DEBUG SenderThread:2023517 [sender.py:send():379] send: files +2025-01-04 21:15:05,294 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_783185f11af79f76b1fa.mp4 with policy now +2025-01-04 21:15:05,294 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:15:05,296 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:15:05,296 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:15:05,297 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:15:05,523 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/jc1mdfnz-media/videos/eval/0_eval_0_783185f11af79f76b1fa.mp4 +2025-01-04 21:15:05,563 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:05,563 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:15:05,563 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/0_eval_0_783185f11af79f76b1fa.mp4 +2025-01-04 21:15:05,563 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media +2025-01-04 21:15:05,563 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval +2025-01-04 21:15:05,563 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos +2025-01-04 21:15:05,828 INFO wandb-upload_0:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/34t5s8my-media/videos/eval/0_eval_0_783185f11af79f76b1fa.mp4 +2025-01-04 21:15:05,945 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:06,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:07,302 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:07,563 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:07,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:08,952 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:09,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:10,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:11,565 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:11,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:12,673 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:12,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:13,565 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:13,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:14,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:15,566 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:15:15,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:16,947 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:17,567 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:17,685 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:17,947 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:18,620 DEBUG SystemMonitor:2023517 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 21:15:18,621 DEBUG SenderThread:2023517 [sender.py:send():379] send: stats +2025-01-04 21:15:18,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:15:18,938 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:15:18,947 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:19,568 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:19,949 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:20,949 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:21,568 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:21,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:22,949 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:23,069 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:23,949 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:24,949 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:25,949 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:26,949 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:27,949 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:28,940 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:28,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:29,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:30,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:31,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:32,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:33,938 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:15:33,938 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:15:33,981 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:34,106 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:34,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:35,951 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:36,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:37,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:38,952 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:39,943 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:39,951 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:40,951 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:41,951 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:42,951 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:43,951 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:44,945 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:44,952 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:45,952 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:46,399 DEBUG SenderThread:2023517 [sender.py:send():379] send: files +2025-01-04 21:15:46,399 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f60d14b1f2240ddad80d.mp4 with policy now +2025-01-04 21:15:46,402 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:15:46,407 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:15:46,407 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:15:46,408 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:15:46,576 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:15:46,577 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/1_eval_1_f60d14b1f2240ddad80d.mp4 +2025-01-04 21:15:46,577 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval +2025-01-04 21:15:46,664 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/wpvbayt0-media/videos/eval/1_eval_1_f60d14b1f2240ddad80d.mp4 +2025-01-04 21:15:46,932 INFO wandb-upload_0:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/vk852tgt-media/videos/eval/1_eval_1_f60d14b1f2240ddad80d.mp4 +2025-01-04 21:15:46,955 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:47,577 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:47,958 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:48,621 DEBUG SenderThread:2023517 [sender.py:send():379] send: stats +2025-01-04 21:15:48,946 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:15:48,946 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:15:48,989 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:49,959 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:50,000 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:50,957 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:51,578 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:51,957 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:52,956 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:53,956 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:54,957 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:55,228 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:15:55,579 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:55,580 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:15:55,812 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/mruyzgn5-events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:15:55,958 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:56,329 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:15:56,330 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:15:56,330 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:15:56,580 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:15:56,957 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:57,580 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:15:57,957 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:58,957 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:15:59,957 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:00,483 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:00,957 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:01,581 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:01,958 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:02,965 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:03,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:16:03,939 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:16:03,981 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:04,965 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:05,965 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:06,055 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:06,965 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:07,966 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:08,966 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:09,966 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:10,974 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:11,947 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:11,969 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:12,969 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:13,976 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:14,977 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:15,586 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:15,977 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:16,977 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:17,430 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:17,977 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:18,622 DEBUG SenderThread:2023517 [sender.py:send():379] send: stats +2025-01-04 21:16:18,939 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:16:18,939 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:16:18,978 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:19,587 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:19,977 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:20,977 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:21,588 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:21,977 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:22,469 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:22,992 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:23,990 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:24,990 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:25,589 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:25,990 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:26,992 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:27,555 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:27,590 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:27,625 DEBUG SenderThread:2023517 [sender.py:send():379] send: files +2025-01-04 21:16:27,626 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_53616642cc0e37a91585.mp4 with policy now +2025-01-04 21:16:27,629 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:16:27,630 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:16:27,630 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:16:27,631 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:16:27,853 INFO wandb-upload_0:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/f06b2bqr-media/videos/eval/2_eval_2_53616642cc0e37a91585.mp4 +2025-01-04 21:16:27,995 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:28,590 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:16:28,591 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/2_eval_2_53616642cc0e37a91585.mp4 +2025-01-04 21:16:28,591 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval +2025-01-04 21:16:28,893 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/zrkhfsep-media/videos/eval/2_eval_2_53616642cc0e37a91585.mp4 +2025-01-04 21:16:28,995 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:29,591 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:29,999 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:31,000 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:32,004 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:32,999 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:33,201 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:33,592 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:33,951 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:16:33,951 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:16:33,997 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:35,008 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:35,593 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:36,006 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:37,001 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:38,001 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:38,951 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:39,008 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:40,007 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:41,002 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:41,162 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:16:41,162 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:16:41,163 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:16:41,595 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:16:41,595 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:16:42,008 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:43,010 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:43,596 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:44,012 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:44,714 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:45,011 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:46,011 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:47,011 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:47,597 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:48,011 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:48,622 DEBUG SenderThread:2023517 [sender.py:send():379] send: stats +2025-01-04 21:16:48,943 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:16:48,943 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:16:49,011 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:50,012 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:50,097 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:51,012 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:52,012 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:53,012 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:54,012 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:55,012 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:55,600 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:16:55,948 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:16:56,012 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:57,012 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:58,013 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:16:59,013 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:00,013 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:00,948 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:01,013 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:02,013 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:03,013 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:03,943 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:17:03,944 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:17:04,013 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:05,013 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:06,013 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:06,043 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:07,024 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:07,604 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:17:07,803 DEBUG SenderThread:2023517 [sender.py:send():379] send: files +2025-01-04 21:17:07,803 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4c459657ba9efcfa48fb.mp4 with policy now +2025-01-04 21:17:07,806 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:17:07,806 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:17:07,806 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:17:07,806 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:17:08,019 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:08,032 INFO wandb-upload_0:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/93ubdiju-media/videos/eval/3_eval_3_4c459657ba9efcfa48fb.mp4 +2025-01-04 21:17:08,604 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:17:08,605 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/3_eval_3_4c459657ba9efcfa48fb.mp4 +2025-01-04 21:17:08,605 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval +2025-01-04 21:17:08,861 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/54j3f85v-media/videos/eval/3_eval_3_4c459657ba9efcfa48fb.mp4 +2025-01-04 21:17:09,024 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:09,605 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:17:10,020 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:11,022 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:11,241 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:12,024 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:13,021 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:13,606 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:17:14,023 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:15,025 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:15,607 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:17:16,028 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:17,024 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:17,069 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:18,030 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:18,623 DEBUG SenderThread:2023517 [sender.py:send():379] send: stats +2025-01-04 21:17:18,949 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:17:18,950 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:17:19,024 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:20,027 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:21,025 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:22,026 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:22,085 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:23,029 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:24,026 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:25,025 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:26,032 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:27,045 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:27,101 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:27,219 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:17:27,219 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:17:27,220 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:17:27,611 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:17:27,611 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:17:27,863 INFO wandb-upload_0:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/eb9z5mk0-events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:17:28,034 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:29,033 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:29,612 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:17:30,031 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:31,029 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:32,029 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:32,360 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:33,029 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:33,613 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:17:33,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:17:33,944 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:17:34,034 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:35,034 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:36,034 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:37,035 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:38,016 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:38,035 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:39,038 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:40,038 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:41,039 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:42,039 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:43,039 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:43,952 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:44,040 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:45,040 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:46,040 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:47,040 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:48,040 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:48,623 DEBUG SenderThread:2023517 [sender.py:send():379] send: stats +2025-01-04 21:17:48,944 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:17:48,944 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:17:49,040 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:49,063 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:50,041 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:51,041 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:52,041 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:53,041 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:54,041 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:54,953 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:17:55,041 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:55,621 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:17:56,041 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:57,041 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:58,042 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:59,051 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:17:59,373 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:17:59,373 DEBUG SenderThread:2023517 [sender.py:send():379] send: files +2025-01-04 21:17:59,373 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_a3c750bc6c1ba4d47759.mp4 with policy now +2025-01-04 21:17:59,374 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:17:59,374 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:17:59,374 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:17:59,622 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:17:59,623 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:17:59,623 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/4_eval_4_a3c750bc6c1ba4d47759.mp4 +2025-01-04 21:17:59,623 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval +2025-01-04 21:17:59,627 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/3ms2sgi5-media/videos/eval/4_eval_4_a3c750bc6c1ba4d47759.mp4 +2025-01-04 21:17:59,881 INFO wandb-upload_0:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/2k81dwu3-media/videos/eval/4_eval_4_a3c750bc6c1ba4d47759.mp4 +2025-01-04 21:18:00,048 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:00,420 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:18:01,054 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:01,623 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:18:02,052 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:03,077 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:03,952 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:18:03,952 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:18:04,059 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:05,064 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:05,624 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:18:05,950 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:18:06,072 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:07,057 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:07,625 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:18:08,055 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:09,063 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:10,070 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:10,272 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:18:10,273 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:10,273 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:10,338 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:18:10,338 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:18:10,524 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:18:10,626 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:18:10,626 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:18:10,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-04 21:18:10,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media +2025-01-04 21:18:10,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table/eval +2025-01-04 21:18:10,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table +2025-01-04 21:18:10,803 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/7hvggdtl-media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-04 21:18:10,877 INFO wandb-upload_0:2023517 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwadsqzmy +2025-01-04 21:18:10,952 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:18:10,962 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:18:11,059 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:11,302 INFO SenderThread:2023517 [sender.py:send_request_log_artifact():1525] logged artifact run-02thhb4i-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE0MzgzOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1NDE3Mg==', 'latestArtifact': None}} +2025-01-04 21:18:11,302 DEBUG SenderThread:2023517 [sender.py:send():379] send: files +2025-01-04 21:18:11,302 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json with policy now +2025-01-04 21:18:11,303 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:18:11,303 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:18:11,303 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,304 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,304 DEBUG SenderThread:2023517 [sender.py:send():379] send: telemetry +2025-01-04 21:18:11,304 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,308 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,308 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,308 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,308 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,309 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,309 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,309 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,309 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,310 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,310 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,310 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,310 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,311 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,311 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,311 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,311 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,312 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,312 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,312 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,312 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:11,313 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:11,313 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:18:11,537 INFO wandb-upload_1:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/48xcex5h-media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-04 21:18:11,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:18:11,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:18:11,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:18:11,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table/eval/total_success_rate_ci_6_4fb6272d24a89dcc8336.table.json +2025-01-04 21:18:11,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table/eval +2025-01-04 21:18:11,652 INFO wandb-upload_0:2023517 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpj9uhk76r +2025-01-04 21:18:11,812 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/zklxcxi0-media/table/eval/total_success_rate_ci_6_4fb6272d24a89dcc8336.table.json +2025-01-04 21:18:12,054 INFO SenderThread:2023517 [sender.py:send_request_log_artifact():1525] logged artifact run-02thhb4i-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE0Mzg1MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1NDE3NA==', 'latestArtifact': None}} +2025-01-04 21:18:12,054 DEBUG SenderThread:2023517 [sender.py:send():379] send: files +2025-01-04 21:18:12,054 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_4fb6272d24a89dcc8336.table.json with policy now +2025-01-04 21:18:12,055 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:18:12,055 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:12,055 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:12,059 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:12,278 INFO wandb-upload_1:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/pws6kvzy-media/table/eval/total_success_rate_ci_6_4fb6272d24a89dcc8336.table.json +2025-01-04 21:18:12,627 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:18:13,059 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:13,558 DEBUG SenderThread:2023517 [sender.py:send():379] send: exit +2025-01-04 21:18:13,558 INFO SenderThread:2023517 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 21:18:13,558 INFO SenderThread:2023517 [sender.py:send_exit():588] handling runtime: 234 +2025-01-04 21:18:13,558 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:13,558 INFO SenderThread:2023517 [sender.py:send_exit():594] send defer +2025-01-04 21:18:13,558 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:13,558 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 21:18:13,559 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:13,559 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 21:18:13,559 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 1 +2025-01-04 21:18:13,559 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:13,559 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 21:18:13,559 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:13,559 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 21:18:13,559 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 2 +2025-01-04 21:18:13,559 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:13,559 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 21:18:13,559 INFO HandlerThread:2023517 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 21:18:13,559 DEBUG SystemMonitor:2023517 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 21:18:13,559 DEBUG SystemMonitor:2023517 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 21:18:13,560 INFO HandlerThread:2023517 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 21:18:13,560 INFO HandlerThread:2023517 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 21:18:13,614 INFO HandlerThread:2023517 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 21:18:13,615 INFO HandlerThread:2023517 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 21:18:13,615 INFO HandlerThread:2023517 [interfaces.py:finish():200] Joined network monitor +2025-01-04 21:18:13,615 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:13,615 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 21:18:13,615 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 3 +2025-01-04 21:18:13,615 DEBUG SenderThread:2023517 [sender.py:send():379] send: stats +2025-01-04 21:18:13,615 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:13,616 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 21:18:13,616 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:13,616 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 21:18:13,616 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 4 +2025-01-04 21:18:13,616 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:13,616 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 21:18:13,628 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:18:13,628 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:18:15,628 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:18:20,278 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:18:20,278 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:18:20,278 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 21:18:20,280 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:20,280 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 21:18:20,280 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 5 +2025-01-04 21:18:20,280 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:18:20,280 DEBUG SenderThread:2023517 [sender.py:send():379] send: history +2025-01-04 21:18:20,280 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:18:20,280 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:20,280 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 21:18:20,281 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:20,281 DEBUG SenderThread:2023517 [sender.py:send():379] send: summary +2025-01-04 21:18:20,281 INFO SenderThread:2023517 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:18:20,282 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:20,282 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 21:18:20,282 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 6 +2025-01-04 21:18:20,282 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:20,282 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 21:18:20,282 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:20,282 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 21:18:20,285 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:18:20,407 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 7 +2025-01-04 21:18:20,407 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:20,407 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 21:18:20,407 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:20,407 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 21:18:20,559 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:18:20,631 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/config.yaml +2025-01-04 21:18:20,631 INFO Thread-12 :2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:18:21,668 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 8 +2025-01-04 21:18:21,668 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:18:21,669 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:21,669 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 21:18:21,669 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:21,669 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 21:18:21,669 INFO SenderThread:2023517 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 21:18:21,671 INFO SenderThread:2023517 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 21:18:21,673 INFO SenderThread:2023517 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 21:18:21,682 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 9 +2025-01-04 21:18:21,682 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:21,682 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 21:18:21,683 DEBUG SenderThread:2023517 [sender.py:send():379] send: artifact +2025-01-04 21:18:21,893 INFO SenderThread:2023517 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 21:18:21,893 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:21,893 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 21:18:21,893 INFO SenderThread:2023517 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 21:18:22,559 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:18:22,632 INFO SenderThread:2023517 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:18:22,632 INFO SenderThread:2023517 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files +2025-01-04 21:18:22,632 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/config.yaml config.yaml +2025-01-04 21:18:22,632 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json wandb-summary.json +2025-01-04 21:18:22,632 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/requirements.txt requirements.txt +2025-01-04 21:18:22,632 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-metadata.json wandb-metadata.json +2025-01-04 21:18:22,632 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log output.log +2025-01-04 21:18:22,633 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/diff.patch diff.patch +2025-01-04 21:18:22,633 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:18:22,640 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 21:18:22,641 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/0_eval_0_783185f11af79f76b1fa.mp4 media/videos/eval/0_eval_0_783185f11af79f76b1fa.mp4 +2025-01-04 21:18:22,641 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/3_eval_3_4c459657ba9efcfa48fb.mp4 media/videos/eval/3_eval_3_4c459657ba9efcfa48fb.mp4 +2025-01-04 21:18:22,641 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/2_eval_2_53616642cc0e37a91585.mp4 media/videos/eval/2_eval_2_53616642cc0e37a91585.mp4 +2025-01-04 21:18:22,641 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/1_eval_1_f60d14b1f2240ddad80d.mp4 media/videos/eval/1_eval_1_f60d14b1f2240ddad80d.mp4 +2025-01-04 21:18:22,641 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/videos/eval/4_eval_4_a3c750bc6c1ba4d47759.mp4 media/videos/eval/4_eval_4_a3c750bc6c1ba4d47759.mp4 +2025-01-04 21:18:22,641 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table/eval/total_success_rate_ci_6_4fb6272d24a89dcc8336.table.json media/table/eval/total_success_rate_ci_6_4fb6272d24a89dcc8336.table.json +2025-01-04 21:18:22,642 INFO SenderThread:2023517 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-04 21:18:22,642 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 10 +2025-01-04 21:18:22,642 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:18:22,643 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:22,643 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 21:18:22,644 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:22,644 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 21:18:22,644 INFO SenderThread:2023517 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:18:22,839 INFO wandb-upload_3:2023517 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/config.yaml +2025-01-04 21:18:22,844 INFO wandb-upload_2:2023517 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/wandb-summary.json +2025-01-04 21:18:22,858 INFO wandb-upload_0:2023517 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/requirements.txt +2025-01-04 21:18:22,912 INFO wandb-upload_1:2023517 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/files/output.log +2025-01-04 21:18:22,948 INFO wandb-upload_4:2023517 [upload_job.py:push():130] Uploaded file /tmp/tmpdjc1tgk8wandb/wdg0y3tm-events.out.tfevents.1736043258.leonmkim-ROG-Strix-G15CS-G15CS.2023350.0 +2025-01-04 21:18:23,148 INFO Thread-11 (_thread_body):2023517 [sender.py:transition_state():614] send defer: 11 +2025-01-04 21:18:23,149 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:23,149 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 21:18:23,149 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:23,149 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 21:18:23,149 INFO SenderThread:2023517 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 21:18:23,150 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 12 +2025-01-04 21:18:23,150 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:23,150 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 21:18:23,150 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:23,151 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 21:18:23,151 INFO SenderThread:2023517 [file_stream.py:finish():601] file stream finish called +2025-01-04 21:18:23,528 INFO SenderThread:2023517 [file_stream.py:finish():605] file stream finish is done +2025-01-04 21:18:23,528 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 13 +2025-01-04 21:18:23,528 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:23,528 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 21:18:23,528 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:23,529 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 21:18:23,529 INFO SenderThread:2023517 [sender.py:transition_state():614] send defer: 14 +2025-01-04 21:18:23,529 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:18:23,529 INFO HandlerThread:2023517 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 21:18:23,529 DEBUG SenderThread:2023517 [sender.py:send():379] send: final +2025-01-04 21:18:23,529 DEBUG SenderThread:2023517 [sender.py:send():379] send: footer +2025-01-04 21:18:23,529 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: defer +2025-01-04 21:18:23,529 INFO SenderThread:2023517 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 21:18:23,530 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:18:23,530 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:18:23,530 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 21:18:23,530 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: server_info +2025-01-04 21:18:23,532 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:18:23,532 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 21:18:23,532 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 21:18:23,533 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:18:23,591 DEBUG SenderThread:2023517 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:18:23,591 INFO MainThread:2023517 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 21:18:23,592 INFO MainThread:2023517 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 21:18:23,592 INFO MainThread:2023517 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 21:18:23,592 DEBUG HandlerThread:2023517 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 21:18:23,592 INFO HandlerThread:2023517 [handler.py:finish():882] shutting down handler +2025-01-04 21:18:24,532 INFO WriterThread:2023517 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/run-02thhb4i.wandb +2025-01-04 21:18:24,591 INFO SenderThread:2023517 [sender.py:finish():1615] shutting down sender +2025-01-04 21:18:24,591 INFO SenderThread:2023517 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:18:24,591 INFO SenderThread:2023517 [file_pusher.py:join():175] waiting for file pusher diff --git a/02thhb4i/wandb/run-20250104_211418-02thhb4i/logs/debug.log b/02thhb4i/wandb/run-20250104_211418-02thhb4i/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7af320536742dd90dd9b01efc89b590ccfd1cbdf --- /dev/null +++ b/02thhb4i/wandb/run-20250104_211418-02thhb4i/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 21:14:18,320 INFO MainThread:2023350 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_setup.py:_flush():76] Configure stats pid to 2023350 +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/settings +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/logs/debug.log +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/wandb/run-20250104_211418-02thhb4i/logs/debug-internal.log +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_init.py:init():569] calling init triggers +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i'} +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_init.py:init():619] starting backend +2025-01-04 21:14:18,321 INFO MainThread:2023350 [wandb_init.py:init():623] setting up manager +2025-01-04 21:14:18,326 INFO MainThread:2023350 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 21:14:18,326 INFO MainThread:2023350 [wandb_init.py:init():631] backend started and connected +2025-01-04 21:14:18,340 INFO MainThread:2023350 [wandb_init.py:init():720] updated telemetry +2025-01-04 21:14:18,347 INFO MainThread:2023350 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 21:14:18,566 INFO MainThread:2023350 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 21:14:18,607 INFO MainThread:2023350 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 21:14:18,607 INFO MainThread:2023350 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 21:14:18,937 INFO MainThread:2023350 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 21:14:18,937 INFO MainThread:2023350 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 21:14:18,937 INFO MainThread:2023350 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 21:14:18,937 INFO MainThread:2023350 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 21:14:18,939 INFO MainThread:2023350 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 21:14:18,939 INFO MainThread:2023350 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/02thhb4i/tb, True +2025-01-04 21:14:23,340 INFO MainThread:2023350 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-04 21:18:24,593 WARNING MsgRouterThr:2023350 [router.py:message_loop():77] message_loop has been closed diff --git a/02thhb4i/wandb/run-20250104_211418-02thhb4i/run-02thhb4i.wandb b/02thhb4i/wandb/run-20250104_211418-02thhb4i/run-02thhb4i.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a1fa6f69e5009bc687c5a21a8691e6d40e518f36 Binary files /dev/null and b/02thhb4i/wandb/run-20250104_211418-02thhb4i/run-02thhb4i.wandb differ diff --git a/29yeoadw/episode_rosbags/episode_0_2025-01-04-20-59-00.bag b/29yeoadw/episode_rosbags/episode_0_2025-01-04-20-59-00.bag new file mode 100644 index 0000000000000000000000000000000000000000..2a29e6c8e367206c0009edf2250823e1f4b74ef7 --- /dev/null +++ b/29yeoadw/episode_rosbags/episode_0_2025-01-04-20-59-00.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2865fce9781615728b2214fdde7dedc7d3e3a9d736eb445e974cda66b228cbd +size 1409281582 diff --git a/29yeoadw/episode_rosbags/episode_1_2025-01-04-20-59-40.bag b/29yeoadw/episode_rosbags/episode_1_2025-01-04-20-59-40.bag new file mode 100644 index 0000000000000000000000000000000000000000..69cefcfb0a220c964d7d9694db247d047c7c8759 --- /dev/null +++ b/29yeoadw/episode_rosbags/episode_1_2025-01-04-20-59-40.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:196427ddc8d84615cdbb4f389ed6da5c90163340e8f5399d5deb23bbd21aed53 +size 1456651880 diff --git a/a4l8onx6/.hydra/config.yaml b/a4l8onx6/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50c4489cc892923b6fde889d218eeedfe242c7ee --- /dev/null +++ b/a4l8onx6/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1922_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/a4l8onx6/.hydra/hydra.yaml b/a4l8onx6/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a5b6e33cd443ac36245fbc2ac1384ee5ed6445c6 --- /dev/null +++ b/a4l8onx6/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/a4l8onx6 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/a4l8onx6/.hydra/overrides.yaml b/a4l8onx6/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/a4l8onx6/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/a4l8onx6/eval_policy.log b/a4l8onx6/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..90cb1c5a9b3221a1dc3382e218817930bda4103b --- /dev/null +++ b/a4l8onx6/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-03 19:51:03,821][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 19:51:03,824][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-03 19:51:04,972][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/k9rmk9dl/.hydra/config.yaml b/k9rmk9dl/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50c4489cc892923b6fde889d218eeedfe242c7ee --- /dev/null +++ b/k9rmk9dl/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1922_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/k9rmk9dl/.hydra/hydra.yaml b/k9rmk9dl/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22983b06c2aa0c00c48ea8b2840bc87bee3a5cff --- /dev/null +++ b/k9rmk9dl/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/k9rmk9dl/.hydra/overrides.yaml b/k9rmk9dl/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/k9rmk9dl/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/k9rmk9dl/episode_rosbags/aligned_depth_to_color_K.npy b/k9rmk9dl/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/k9rmk9dl/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/k9rmk9dl/episode_rosbags/cam_tf_world.npy b/k9rmk9dl/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/k9rmk9dl/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/k9rmk9dl/episode_rosbags/color_K.npy b/k9rmk9dl/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/k9rmk9dl/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/k9rmk9dl/episode_rosbags/depth_K.npy b/k9rmk9dl/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/k9rmk9dl/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/k9rmk9dl/episode_rosbags/episode_0_2025-01-03-19-58-06.bag b/k9rmk9dl/episode_rosbags/episode_0_2025-01-03-19-58-06.bag new file mode 100644 index 0000000000000000000000000000000000000000..1cb2cd6f4d29777eb53cae461e51d266a25e1c83 --- /dev/null +++ b/k9rmk9dl/episode_rosbags/episode_0_2025-01-03-19-58-06.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5e243e7533aa97e0bd0021b00560af159bd3bb972ea5e67d2ce7eb82338f0fe +size 1411086989 diff --git a/k9rmk9dl/episode_rosbags/episode_1_2025-01-03-19-58-48.bag b/k9rmk9dl/episode_rosbags/episode_1_2025-01-03-19-58-48.bag new file mode 100644 index 0000000000000000000000000000000000000000..0997205a4d38190d91a9d4505a03f19ea447cf01 --- /dev/null +++ b/k9rmk9dl/episode_rosbags/episode_1_2025-01-03-19-58-48.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e99e8e3e4be4fa8efe0e13593a5f8d415d3ac45402e78f4e17291222bc898fcb +size 1410914537 diff --git a/k9rmk9dl/episode_rosbags/episode_2_2025-01-03-19-59-30.bag b/k9rmk9dl/episode_rosbags/episode_2_2025-01-03-19-59-30.bag new file mode 100644 index 0000000000000000000000000000000000000000..562bbd5aa3dec0aeb3fe3153d2469b045aaa01e5 --- /dev/null +++ b/k9rmk9dl/episode_rosbags/episode_2_2025-01-03-19-59-30.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0588abf0ca287dfea3e227f79bf96e2015799e41b7437dcdbae24fc80b27c900 +size 1417471165 diff --git a/k9rmk9dl/episode_rosbags/episode_3_2025-01-03-20-00-12.bag b/k9rmk9dl/episode_rosbags/episode_3_2025-01-03-20-00-12.bag new file mode 100644 index 0000000000000000000000000000000000000000..8aac7db5b762bc1132a00e4a01437a672ad1f543 --- /dev/null +++ b/k9rmk9dl/episode_rosbags/episode_3_2025-01-03-20-00-12.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:815407d42aaaa8f3ebdc92a182bc6b29852817289d67e139a4002d896232f23f +size 1412595993 diff --git a/k9rmk9dl/episode_rosbags/episode_4_2025-01-03-20-00-55.bag b/k9rmk9dl/episode_rosbags/episode_4_2025-01-03-20-00-55.bag new file mode 100644 index 0000000000000000000000000000000000000000..d7b65fbeb2140d5cac13e097a4c0abc6afd2433a --- /dev/null +++ b/k9rmk9dl/episode_rosbags/episode_4_2025-01-03-20-00-55.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d5d43d60e4bb9eb5477627b7a1832f3ac2c70cd2f649925d445929f40799f2b +size 1408165142 diff --git a/k9rmk9dl/eval_robot.log b/k9rmk9dl/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..d4b4f8709b8adde9a26e55b3ef9a3e29c1c52493 --- /dev/null +++ b/k9rmk9dl/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 19:57:42,291][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 19:57:42,294][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/k9rmk9dl/eval_video/0_eval.mp4 b/k9rmk9dl/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9ba53df1dafa4d68cf7de78510b451247e8d9ab3 --- /dev/null +++ b/k9rmk9dl/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:242d6ca502a6c74dce290cd496f05a21c3f2e89c852b8b58e66ac52de0312c20 +size 1133413 diff --git a/k9rmk9dl/eval_video/1_eval.mp4 b/k9rmk9dl/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e05de9cf724c29e775f18adf268ce9f30d55bfb2 --- /dev/null +++ b/k9rmk9dl/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92e676c219a8081d0b991ffddbce20df13b10f2d111101e9c90619203bf8ad86 +size 1132108 diff --git a/k9rmk9dl/eval_video/2_eval.mp4 b/k9rmk9dl/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c5ac079a748b4e5b0ddbb447b8e4e861a7208beb --- /dev/null +++ b/k9rmk9dl/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95cc898a615ff7a378e0626fdf7a7061bedc3c968cba3ab46b0ec02a4d7f70b1 +size 1138478 diff --git a/k9rmk9dl/eval_video/3_eval.mp4 b/k9rmk9dl/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..08d410b37a46df90f14dae2300384fe27d520890 --- /dev/null +++ b/k9rmk9dl/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea9f563ddcabcb5060b655b365ba0570d2049fe2c325e2d60a77a5f182ca5781 +size 1132960 diff --git a/k9rmk9dl/eval_video/4_eval.mp4 b/k9rmk9dl/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..132216c3c434692bd53eebed6cfdd3f8591fc684 --- /dev/null +++ b/k9rmk9dl/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9f7cad01419898a5561dea8588234d342a92c72fd87792dd7282118439bb0ed +size 1144971 diff --git a/k9rmk9dl/tb/events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 b/k9rmk9dl/tb/events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 new file mode 100644 index 0000000000000000000000000000000000000000..5ac0cfdc190dd28adb5afa1deffadb9ca8732863 --- /dev/null +++ b/k9rmk9dl/tb/events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da093e9fe99ab0439d0bbf34278014a25470bc88091bbed9622ac061e5d6bd2b +size 6539 diff --git a/k9rmk9dl/wandb/debug-internal.log b/k9rmk9dl/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..99fdc0d6ee12d456f2b7cf6dd659a1af5b9ae33f --- /dev/null +++ b/k9rmk9dl/wandb/debug-internal.log @@ -0,0 +1,716 @@ +2025-01-03 19:57:48,046 INFO StreamThr :1960299 [internal.py:wandb_internal():85] W&B internal server running at pid: 1960299, started at: 2025-01-03 19:57:48.046468 +2025-01-03 19:57:48,048 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status +2025-01-03 19:57:48,060 INFO WriterThread:1960299 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/run-k9rmk9dl.wandb +2025-01-03 19:57:48,060 DEBUG SenderThread:1960299 [sender.py:send():379] send: header +2025-01-03 19:57:48,067 DEBUG SenderThread:1960299 [sender.py:send():379] send: run +2025-01-03 19:57:48,343 INFO SenderThread:1960299 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files +2025-01-03 19:57:48,343 INFO SenderThread:1960299 [sender.py:_start_run_threads():1188] run started: k9rmk9dl with start time 1735952268.047168 +2025-01-03 19:57:48,357 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 19:57:48,357 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: check_version +2025-01-03 19:57:48,429 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 19:57:48,434 DEBUG HandlerThread:1960299 [system_info.py:__init__():26] System info init +2025-01-03 19:57:48,434 DEBUG HandlerThread:1960299 [system_info.py:__init__():41] System info init done +2025-01-03 19:57:48,434 INFO HandlerThread:1960299 [system_monitor.py:start():194] Starting system monitor +2025-01-03 19:57:48,434 INFO SystemMonitor:1960299 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 19:57:48,434 INFO HandlerThread:1960299 [system_monitor.py:probe():214] Collecting system info +2025-01-03 19:57:48,434 INFO SystemMonitor:1960299 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 19:57:48,435 INFO SystemMonitor:1960299 [interfaces.py:start():188] Started disk monitoring +2025-01-03 19:57:48,436 INFO SystemMonitor:1960299 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 19:57:48,436 INFO SystemMonitor:1960299 [interfaces.py:start():188] Started memory monitoring +2025-01-03 19:57:48,437 INFO SystemMonitor:1960299 [interfaces.py:start():188] Started network monitoring +2025-01-03 19:57:48,452 DEBUG HandlerThread:1960299 [system_info.py:probe():152] Probing system +2025-01-03 19:57:48,454 DEBUG HandlerThread:1960299 [system_info.py:_probe_git():137] Probing git +2025-01-03 19:57:48,459 DEBUG HandlerThread:1960299 [system_info.py:_probe_git():145] Probing git done +2025-01-03 19:57:48,459 DEBUG HandlerThread:1960299 [system_info.py:probe():200] Probing system done +2025-01-03 19:57:48,459 DEBUG HandlerThread:1960299 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T00:57:48.452488', 'startedAt': '2025-01-04T00:57:48.037416', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3936.315625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.289, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 711.108470916748}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 19:57:48,459 INFO HandlerThread:1960299 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 19:57:48,459 INFO HandlerThread:1960299 [system_monitor.py:probe():227] Publishing system info +2025-01-03 19:57:48,459 DEBUG HandlerThread:1960299 [system_info.py:_save_code():44] Saving code +2025-01-03 19:57:48,460 DEBUG HandlerThread:1960299 [system_info.py:_save_code():67] Saving code done +2025-01-03 19:57:48,460 DEBUG HandlerThread:1960299 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 19:57:48,524 DEBUG HandlerThread:1960299 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 19:57:48,524 INFO HandlerThread:1960299 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 19:57:48,528 DEBUG SenderThread:1960299 [sender.py:send():379] send: files +2025-01-03 19:57:48,528 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 19:57:48,528 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 19:57:48,528 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 19:57:48,763 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 19:57:48,763 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: python_packages +2025-01-03 19:57:48,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 19:57:48,764 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 19:57:48,765 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:48,766 INFO HandlerThread:1960299 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/tb" + save: true +} +control { + relay_id: "dfc456c70ff745de8fb44313dae7533c" +} +_info { + stream_id: "k9rmk9dl" +} + +2025-01-03 19:57:48,766 DEBUG HandlerThread:1960299 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 19:57:48,800 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/1mia9ah7-wandb-metadata.json +2025-01-03 19:57:48,806 INFO wandb-upload_2:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/2oqurbl6-diff.patch +2025-01-03 19:57:48,834 DEBUG SenderThread:1960299 [sender.py:send():379] send: telemetry +2025-01-03 19:57:48,835 DEBUG SenderThread:1960299 [sender.py:send():379] send: tbrecord +2025-01-03 19:57:48,835 DEBUG SenderThread:1960299 [sender.py:send():379] send: files +2025-01-03 19:57:48,835 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 with policy live +2025-01-03 19:57:48,836 INFO wandb-upload_1:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/9ev08r4b-code/FISH/eval_robot.py +2025-01-03 19:57:49,055 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/z46mxsa1-events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 19:57:49,345 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/diff.patch +2025-01-03 19:57:49,345 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 19:57:49,345 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-metadata.json +2025-01-03 19:57:49,345 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/requirements.txt +2025-01-03 19:57:49,345 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:57:49,346 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/code/FISH/eval_robot.py +2025-01-03 19:57:49,346 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/code +2025-01-03 19:57:49,346 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/code/FISH +2025-01-03 19:57:49,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:50,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:51,344 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:57:51,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:52,618 DEBUG SenderThread:1960299 [sender.py:send():379] send: config +2025-01-03 19:57:52,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:53,345 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:57:53,625 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:57:53,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:54,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:55,345 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:57:55,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:56,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:57,346 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:57:57,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:58,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:57:58,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:59,347 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:57:59,765 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:00,765 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:01,347 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:01,765 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:02,766 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:03,763 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 19:58:03,763 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 19:58:03,766 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:03,904 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:04,766 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:05,348 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:05,766 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:06,766 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:07,766 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:08,766 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:09,350 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:09,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:09,766 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:10,767 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:11,767 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:12,767 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:13,767 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:14,767 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:14,769 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:15,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:16,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:17,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:18,763 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 19:58:18,763 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 19:58:18,805 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:19,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:19,843 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:20,353 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/config.yaml +2025-01-03 19:58:20,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:21,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:22,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:23,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:24,771 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:25,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:25,771 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:26,771 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:27,771 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:28,772 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:29,772 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:30,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:30,772 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:31,772 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:32,772 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:33,365 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:33,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 19:58:33,764 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 19:58:33,772 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:34,347 DEBUG SenderThread:1960299 [sender.py:send():379] send: files +2025-01-03 19:58:34,347 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_242d6ca502a6c74dce29.mp4 with policy now +2025-01-03 19:58:34,353 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 19:58:34,354 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 19:58:34,354 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 19:58:34,354 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 19:58:34,366 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 19:58:34,366 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/0_eval_0_242d6ca502a6c74dce29.mp4 +2025-01-03 19:58:34,366 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media +2025-01-03 19:58:34,366 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos +2025-01-03 19:58:34,366 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval +2025-01-03 19:58:34,604 INFO wandb-upload_1:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/9v4xf6g9-media/videos/eval/0_eval_0_242d6ca502a6c74dce29.mp4 +2025-01-03 19:58:34,773 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:34,847 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/klc7xu2t-media/videos/eval/0_eval_0_242d6ca502a6c74dce29.mp4 +2025-01-03 19:58:35,366 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:35,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:36,361 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:36,773 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:37,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:38,773 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:39,367 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:39,773 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:40,813 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:41,774 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:41,814 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:42,774 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:43,369 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:43,774 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:44,774 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:45,369 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 19:58:45,774 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:46,775 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:47,370 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:47,637 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:47,774 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:48,437 DEBUG SystemMonitor:1960299 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 19:58:48,438 DEBUG SenderThread:1960299 [sender.py:send():379] send: stats +2025-01-03 19:58:48,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 19:58:48,764 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 19:58:48,775 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:49,371 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:49,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:50,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:51,372 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:51,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:52,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:52,897 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:53,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:54,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:55,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:56,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:57,374 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:57,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:58,322 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:58,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:59,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:00,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:01,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:02,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:03,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 19:59:03,764 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 19:59:03,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:03,823 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:04,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:05,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:06,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:07,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:08,778 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:09,772 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:09,778 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:10,778 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:11,778 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:12,778 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:13,778 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:14,779 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:15,266 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:15,779 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:16,536 DEBUG SenderThread:1960299 [sender.py:send():379] send: files +2025-01-03 19:59:16,536 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_92e676c219a8081d0b99.mp4 with policy now +2025-01-03 19:59:16,540 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 19:59:16,540 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 19:59:16,541 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 19:59:16,541 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 19:59:16,783 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:16,820 INFO wandb-upload_1:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/3ywqzj5d-media/videos/eval/1_eval_1_92e676c219a8081d0b99.mp4 +2025-01-03 19:59:17,380 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 19:59:17,380 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:59:17,380 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/1_eval_1_92e676c219a8081d0b99.mp4 +2025-01-03 19:59:17,380 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval +2025-01-03 19:59:17,656 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/jjudmfxq-media/videos/eval/1_eval_1_92e676c219a8081d0b99.mp4 +2025-01-03 19:59:17,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:18,438 DEBUG SenderThread:1960299 [sender.py:send():379] send: stats +2025-01-03 19:59:18,767 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 19:59:18,767 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 19:59:18,813 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:19,381 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:59:19,785 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:20,283 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:20,796 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:21,381 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:59:21,793 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:22,792 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:23,784 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:24,784 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:25,383 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:59:25,772 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:25,785 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:26,784 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:27,383 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 19:59:27,644 INFO wandb-upload_1:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/7s6ixnoj-events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 19:59:27,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:28,170 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 19:59:28,170 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 19:59:28,171 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 19:59:28,384 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 19:59:28,785 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:29,384 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:59:29,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:30,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:30,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:31,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:32,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:33,385 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:59:33,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 19:59:33,764 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 19:59:33,805 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:34,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:35,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:35,855 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:36,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:37,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:38,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:39,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:40,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:41,773 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:41,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:42,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:43,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:44,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:45,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:46,774 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:46,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:47,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:48,439 DEBUG SenderThread:1960299 [sender.py:send():379] send: stats +2025-01-03 19:59:48,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 19:59:48,764 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 19:59:48,805 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:49,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:50,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:51,788 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:51,836 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:52,788 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:53,788 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:54,788 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:55,788 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:56,788 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:57,774 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:57,788 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:58,793 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:59,384 DEBUG SenderThread:1960299 [sender.py:send():379] send: files +2025-01-03 19:59:59,384 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_95cc898a615ff7a378e0.mp4 with policy now +2025-01-03 19:59:59,387 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 19:59:59,387 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 19:59:59,388 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 19:59:59,388 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 19:59:59,394 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 19:59:59,394 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:59:59,394 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/2_eval_2_95cc898a615ff7a378e0.mp4 +2025-01-03 19:59:59,394 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval +2025-01-03 19:59:59,634 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/crmdl6ea-media/videos/eval/2_eval_2_95cc898a615ff7a378e0.mp4 +2025-01-03 19:59:59,797 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:59,874 INFO wandb-upload_1:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/zpt8gf3d-media/videos/eval/2_eval_2_95cc898a615ff7a378e0.mp4 +2025-01-03 20:00:00,800 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:01,394 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:01,804 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:02,799 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:03,231 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:03,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:00:03,771 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:00:03,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:04,802 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:05,396 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:05,808 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:06,808 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:07,397 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:07,806 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:08,665 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:08,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:09,002 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 20:00:09,002 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:00:09,003 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:00:09,397 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 20:00:09,398 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 20:00:09,818 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:10,810 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:11,398 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:11,814 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:12,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:13,753 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:13,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:14,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:15,399 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:15,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:16,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:17,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:18,439 DEBUG SenderThread:1960299 [sender.py:send():379] send: stats +2025-01-03 20:00:18,766 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:00:18,767 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:00:18,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:18,832 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:19,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:20,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:21,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:22,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:23,808 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:24,775 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:24,808 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:25,403 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:25,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:26,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:27,810 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:28,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:29,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:30,775 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:30,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:31,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:32,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:33,769 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:00:33,769 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:00:33,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:34,810 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:35,810 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:35,843 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:36,810 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:37,810 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:38,810 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:39,811 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:40,828 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:40,960 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:41,005 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:00:41,005 DEBUG SenderThread:1960299 [sender.py:send():379] send: files +2025-01-03 20:00:41,006 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ea9f563ddcabcb5060b6.mp4 with policy now +2025-01-03 20:00:41,006 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 20:00:41,006 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:00:41,007 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:00:41,408 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 20:00:41,408 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:41,408 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/3_eval_3_ea9f563ddcabcb5060b6.mp4 +2025-01-03 20:00:41,408 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval +2025-01-03 20:00:41,551 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/gjb4z54g-media/videos/eval/3_eval_3_ea9f563ddcabcb5060b6.mp4 +2025-01-03 20:00:41,793 INFO wandb-upload_1:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/axwvln0n-media/videos/eval/3_eval_3_ea9f563ddcabcb5060b6.mp4 +2025-01-03 20:00:41,826 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:42,826 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:43,409 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:43,828 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:44,822 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:45,835 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:46,834 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:46,893 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:47,410 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:47,821 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:48,439 DEBUG SenderThread:1960299 [sender.py:send():379] send: stats +2025-01-03 20:00:48,775 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:00:48,776 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:00:48,824 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:49,411 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:49,831 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:50,827 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:51,829 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:51,902 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:52,063 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 20:00:52,063 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:00:52,064 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:00:52,412 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 20:00:52,412 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 20:00:52,680 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/2oafbqye-events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 20:00:52,830 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:53,412 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:53,829 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:54,830 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:55,413 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:55,834 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:56,830 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:57,326 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:57,414 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:57,830 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:58,831 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:59,414 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:59,831 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:00,831 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:01,831 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:02,831 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:03,301 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:03,416 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:01:03,768 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:01:03,769 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:01:03,832 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:04,832 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:05,832 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:06,832 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:07,832 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:08,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:08,832 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:09,832 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:10,832 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:11,834 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:12,836 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:13,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:13,835 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:14,835 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:15,836 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:16,837 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:17,837 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:18,440 DEBUG SenderThread:1960299 [sender.py:send():379] send: stats +2025-01-03 20:01:18,768 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:01:18,769 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:01:18,837 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:18,843 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:19,837 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:20,837 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:21,837 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:22,838 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:23,869 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:23,869 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:24,538 DEBUG SenderThread:1960299 [sender.py:send():379] send: files +2025-01-03 20:01:24,538 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_c9f7cad01419898a5561.mp4 with policy now +2025-01-03 20:01:24,544 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:01:24,544 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 20:01:24,544 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:24,544 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:24,790 INFO wandb-upload_1:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/io18ifql-media/videos/eval/4_eval_4_c9f7cad01419898a5561.mp4 +2025-01-03 20:01:24,839 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:25,423 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 20:01:25,423 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:01:25,423 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/4_eval_4_c9f7cad01419898a5561.mp4 +2025-01-03 20:01:25,424 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval +2025-01-03 20:01:25,700 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/nowm3nn2-media/videos/eval/4_eval_4_c9f7cad01419898a5561.mp4 +2025-01-03 20:01:25,855 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:26,855 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:27,856 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:28,853 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:29,425 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:01:29,797 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:29,857 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:30,861 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:31,856 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:32,857 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:33,426 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:01:33,791 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:01:33,791 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:01:33,863 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:34,875 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:34,907 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:35,880 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:36,865 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:37,866 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:38,064 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:01:38,064 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:01:38,365 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:01:38,428 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 20:01:38,428 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 20:01:38,428 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media +2025-01-03 20:01:38,428 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table +2025-01-03 20:01:38,428 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table/eval +2025-01-03 20:01:38,536 INFO wandb-upload_1:1960299 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpqqw88j0_ +2025-01-03 20:01:38,631 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/5ok435hm-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 20:01:38,718 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:01:38,732 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:01:38,860 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:38,940 INFO SenderThread:1960299 [sender.py:send_request_log_artifact():1525] logged artifact run-k9rmk9dl-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzgxNjU4Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY2NjYwMg==', 'latestArtifact': None}} +2025-01-03 20:01:38,940 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 20:01:38,940 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,941 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,941 DEBUG SenderThread:1960299 [sender.py:send():379] send: files +2025-01-03 20:01:38,941 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-03 20:01:38,941 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 20:01:38,941 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,942 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,942 DEBUG SenderThread:1960299 [sender.py:send():379] send: telemetry +2025-01-03 20:01:38,942 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,942 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,942 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,943 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,943 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,945 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,945 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,945 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,945 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,946 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,946 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,946 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,946 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,947 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,947 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,947 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,947 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,948 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,948 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,948 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,948 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,949 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,949 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:01:39,203 INFO wandb-upload_2:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/wpw1wp1e-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 20:01:39,285 INFO wandb-upload_0:1960299 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpj1x3hdx1 +2025-01-03 20:01:39,428 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 20:01:39,428 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:01:39,428 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 20:01:39,429 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json +2025-01-03 20:01:39,429 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table/eval +2025-01-03 20:01:39,676 INFO SenderThread:1960299 [sender.py:send_request_log_artifact():1525] logged artifact run-k9rmk9dl-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzgxNjYwNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY2NjYwNQ==', 'latestArtifact': None}} +2025-01-03 20:01:39,676 DEBUG SenderThread:1960299 [sender.py:send():379] send: files +2025-01-03 20:01:39,676 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json with policy now +2025-01-03 20:01:39,677 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 20:01:39,677 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:39,677 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:39,853 INFO wandb-upload_1:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/d8h05qo4-media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json +2025-01-03 20:01:39,860 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:40,118 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/2q7wo8nj-media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json +2025-01-03 20:01:40,327 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:40,373 DEBUG SenderThread:1960299 [sender.py:send():379] send: exit +2025-01-03 20:01:40,373 INFO SenderThread:1960299 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 20:01:40,374 INFO SenderThread:1960299 [sender.py:send_exit():588] handling runtime: 231 +2025-01-03 20:01:40,374 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:40,374 INFO SenderThread:1960299 [sender.py:send_exit():594] send defer +2025-01-03 20:01:40,374 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:40,374 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 20:01:40,374 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:40,374 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 20:01:40,375 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 1 +2025-01-03 20:01:40,375 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:40,375 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 20:01:40,375 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:40,375 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 20:01:40,375 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 2 +2025-01-03 20:01:40,375 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:40,375 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 20:01:40,375 INFO HandlerThread:1960299 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 20:01:40,375 INFO HandlerThread:1960299 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 20:01:40,376 DEBUG SystemMonitor:1960299 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 20:01:40,376 DEBUG SystemMonitor:1960299 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 20:01:40,376 INFO HandlerThread:1960299 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 20:01:40,429 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 20:01:40,450 INFO HandlerThread:1960299 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 20:01:40,451 INFO HandlerThread:1960299 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 20:01:40,451 INFO HandlerThread:1960299 [interfaces.py:finish():200] Joined network monitor +2025-01-03 20:01:40,451 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:40,451 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 20:01:40,451 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 3 +2025-01-03 20:01:40,451 DEBUG SenderThread:1960299 [sender.py:send():379] send: stats +2025-01-03 20:01:40,452 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:40,452 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 20:01:40,452 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:40,452 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 20:01:40,452 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 4 +2025-01-03 20:01:40,452 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:40,452 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 20:01:41,429 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:01:47,126 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:01:47,127 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:47,127 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 20:01:47,128 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:47,128 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 20:01:47,128 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 5 +2025-01-03 20:01:47,128 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:01:47,128 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 20:01:47,128 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:47,128 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:47,129 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 20:01:47,129 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:47,129 DEBUG SenderThread:1960299 [sender.py:send():379] send: summary +2025-01-03 20:01:47,130 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:47,130 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:47,130 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 20:01:47,130 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 6 +2025-01-03 20:01:47,130 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:47,130 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 20:01:47,130 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:47,130 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 20:01:47,134 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:47,239 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 7 +2025-01-03 20:01:47,240 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:47,240 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 20:01:47,240 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:47,240 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 20:01:47,375 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:01:47,432 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 20:01:47,432 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/config.yaml +2025-01-03 20:01:49,247 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 8 +2025-01-03 20:01:49,247 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:01:49,247 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:49,247 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 20:01:49,248 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:49,248 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 20:01:49,248 INFO SenderThread:1960299 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 20:01:49,250 INFO SenderThread:1960299 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 20:01:49,252 INFO SenderThread:1960299 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 20:01:49,260 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 9 +2025-01-03 20:01:49,260 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:49,260 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 20:01:49,260 DEBUG SenderThread:1960299 [sender.py:send():379] send: artifact +2025-01-03 20:01:49,376 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:01:49,433 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:01:49,466 INFO SenderThread:1960299 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzgxMTc2Nw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzgxMTc2Nw==', 'versionIndex': 405}}} +2025-01-03 20:01:49,466 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:49,466 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 20:01:49,466 INFO SenderThread:1960299 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 20:01:50,433 INFO SenderThread:1960299 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files +2025-01-03 20:01:50,434 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/config.yaml config.yaml +2025-01-03 20:01:50,434 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 20:01:50,434 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json wandb-summary.json +2025-01-03 20:01:50,434 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/requirements.txt requirements.txt +2025-01-03 20:01:50,434 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-metadata.json wandb-metadata.json +2025-01-03 20:01:50,434 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log output.log +2025-01-03 20:01:50,434 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/diff.patch diff.patch +2025-01-03 20:01:50,436 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 20:01:50,443 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/3_eval_3_ea9f563ddcabcb5060b6.mp4 media/videos/eval/3_eval_3_ea9f563ddcabcb5060b6.mp4 +2025-01-03 20:01:50,443 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/4_eval_4_c9f7cad01419898a5561.mp4 media/videos/eval/4_eval_4_c9f7cad01419898a5561.mp4 +2025-01-03 20:01:50,443 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/1_eval_1_92e676c219a8081d0b99.mp4 media/videos/eval/1_eval_1_92e676c219a8081d0b99.mp4 +2025-01-03 20:01:50,443 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/0_eval_0_242d6ca502a6c74dce29.mp4 media/videos/eval/0_eval_0_242d6ca502a6c74dce29.mp4 +2025-01-03 20:01:50,443 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/2_eval_2_95cc898a615ff7a378e0.mp4 media/videos/eval/2_eval_2_95cc898a615ff7a378e0.mp4 +2025-01-03 20:01:50,444 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 20:01:50,444 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json +2025-01-03 20:01:50,444 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 10 +2025-01-03 20:01:50,444 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:01:50,445 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:50,445 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 20:01:50,445 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:50,445 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 20:01:50,445 INFO SenderThread:1960299 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:01:50,615 INFO wandb-upload_4:1960299 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 20:01:50,617 INFO wandb-upload_1:1960299 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/config.yaml +2025-01-03 20:01:50,670 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/requirements.txt +2025-01-03 20:01:50,683 INFO wandb-upload_2:1960299 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:01:50,728 INFO wandb-upload_3:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/cff81489-events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 20:01:50,928 INFO Thread-11 (_thread_body):1960299 [sender.py:transition_state():614] send defer: 11 +2025-01-03 20:01:50,928 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:50,928 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 20:01:50,929 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:50,929 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 20:01:50,929 INFO SenderThread:1960299 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 20:01:50,930 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 12 +2025-01-03 20:01:50,930 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:50,930 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 20:01:50,930 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:50,930 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 20:01:50,930 INFO SenderThread:1960299 [file_stream.py:finish():601] file stream finish called +2025-01-03 20:01:51,006 INFO SenderThread:1960299 [file_stream.py:finish():605] file stream finish is done +2025-01-03 20:01:51,006 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 13 +2025-01-03 20:01:51,006 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:51,006 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 20:01:51,006 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:51,006 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 20:01:51,007 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 14 +2025-01-03 20:01:51,007 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:51,007 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 20:01:51,007 DEBUG SenderThread:1960299 [sender.py:send():379] send: final +2025-01-03 20:01:51,007 DEBUG SenderThread:1960299 [sender.py:send():379] send: footer +2025-01-03 20:01:51,007 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:51,007 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 20:01:51,008 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:01:51,008 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 20:01:51,008 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:01:51,008 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:01:51,008 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 20:01:51,009 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: server_info +2025-01-03 20:01:51,009 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 20:01:51,011 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:51,069 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:01:51,069 INFO MainThread:1960299 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 20:01:51,069 INFO MainThread:1960299 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 20:01:51,070 INFO MainThread:1960299 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 20:01:51,070 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 20:01:51,070 INFO HandlerThread:1960299 [handler.py:finish():882] shutting down handler +2025-01-03 20:01:52,008 INFO WriterThread:1960299 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/run-k9rmk9dl.wandb +2025-01-03 20:01:52,069 INFO SenderThread:1960299 [sender.py:finish():1615] shutting down sender +2025-01-03 20:01:52,069 INFO SenderThread:1960299 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:01:52,069 INFO SenderThread:1960299 [file_pusher.py:join():175] waiting for file pusher diff --git a/k9rmk9dl/wandb/debug.log b/k9rmk9dl/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bff87eabeed784a8986e4f6485740c01c29716f0 --- /dev/null +++ b/k9rmk9dl/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 19:57:48,040 INFO MainThread:1960134 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 19:57:48,040 INFO MainThread:1960134 [wandb_setup.py:_flush():76] Configure stats pid to 1960134 +2025-01-03 19:57:48,040 INFO MainThread:1960134 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/settings +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/logs/debug.log +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/logs/debug-internal.log +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_init.py:init():569] calling init triggers +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl'} +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_init.py:init():619] starting backend +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_init.py:init():623] setting up manager +2025-01-03 19:57:48,045 INFO MainThread:1960134 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 19:57:48,047 INFO MainThread:1960134 [wandb_init.py:init():631] backend started and connected +2025-01-03 19:57:48,059 INFO MainThread:1960134 [wandb_init.py:init():720] updated telemetry +2025-01-03 19:57:48,064 INFO MainThread:1960134 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 19:57:48,357 INFO MainThread:1960134 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 19:57:48,423 INFO MainThread:1960134 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 19:57:48,424 INFO MainThread:1960134 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 19:57:48,763 INFO MainThread:1960134 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 19:57:48,764 INFO MainThread:1960134 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 19:57:48,764 INFO MainThread:1960134 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 19:57:48,764 INFO MainThread:1960134 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 19:57:48,765 INFO MainThread:1960134 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 19:57:48,765 INFO MainThread:1960134 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/tb, True +2025-01-03 19:57:52,617 INFO MainThread:1960134 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-03 20:01:52,072 WARNING MsgRouterThr:1960134 [router.py:message_loop():77] message_loop has been closed diff --git a/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/code/FISH/eval_robot.py b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/config.yaml b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..147023e9f8e8ee7ede662a7354f0740cea28cd50 --- /dev/null +++ b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1922_1' +wandb_notes: + desc: null + value: 1922_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735952268 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/diff.patch b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..99bc0b0fbb6fb7029b65118694020068d57ffc3a --- /dev/null +++ b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/diff.patch @@ -0,0 +1,50 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..af822a0 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,14 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++wandb_run_id: '1922_1' ++# wandb_run_id: '1922_2' + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..29dc254 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,8 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ '1922_1', ++ '1922_2', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json new file mode 100644 index 0000000000000000000000000000000000000000..5a592aac7e97094d41fce54046633e127c09be66 --- /dev/null +++ b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[10, 10, 1.0, 0.9892721175168917, 1, 0.7146403789090983, 1, 0.7492533623673705, 1, 1, 1]]} \ No newline at end of file diff --git a/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/0_eval_0_242d6ca502a6c74dce29.mp4 b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/0_eval_0_242d6ca502a6c74dce29.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9ba53df1dafa4d68cf7de78510b451247e8d9ab3 --- /dev/null +++ b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/0_eval_0_242d6ca502a6c74dce29.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:242d6ca502a6c74dce290cd496f05a21c3f2e89c852b8b58e66ac52de0312c20 +size 1133413 diff --git a/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/1_eval_1_92e676c219a8081d0b99.mp4 b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/1_eval_1_92e676c219a8081d0b99.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e05de9cf724c29e775f18adf268ce9f30d55bfb2 --- /dev/null +++ b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/1_eval_1_92e676c219a8081d0b99.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92e676c219a8081d0b991ffddbce20df13b10f2d111101e9c90619203bf8ad86 +size 1132108 diff --git a/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/2_eval_2_95cc898a615ff7a378e0.mp4 b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/2_eval_2_95cc898a615ff7a378e0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c5ac079a748b4e5b0ddbb447b8e4e861a7208beb --- /dev/null +++ b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/2_eval_2_95cc898a615ff7a378e0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95cc898a615ff7a378e0626fdf7a7061bedc3c968cba3ab46b0ec02a4d7f70b1 +size 1138478 diff --git a/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/3_eval_3_ea9f563ddcabcb5060b6.mp4 b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/3_eval_3_ea9f563ddcabcb5060b6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..08d410b37a46df90f14dae2300384fe27d520890 --- /dev/null +++ b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/3_eval_3_ea9f563ddcabcb5060b6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea9f563ddcabcb5060b655b365ba0570d2049fe2c325e2d60a77a5f182ca5781 +size 1132960 diff --git a/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/4_eval_4_c9f7cad01419898a5561.mp4 b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/4_eval_4_c9f7cad01419898a5561.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..132216c3c434692bd53eebed6cfdd3f8591fc684 --- /dev/null +++ b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/4_eval_4_c9f7cad01419898a5561.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9f7cad01419898a5561dea8588234d342a92c72fd87792dd7282118439bb0ed +size 1144971 diff --git a/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ed400d85ba1887c0ab2dbf66fa7f04e88944217a --- /dev/null +++ b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log @@ -0,0 +1,276 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735952272.618362]: resetting environment +[INFO] [1735952272.621650]: cleared current plan +[INFO] [1735952272.622787]: moving to home +[INFO] [1735952273.625203]: reached home +[INFO] [1735952273.626463]: reset action history +[INFO] [1735952276.334230]: environment reset +Starting episode 0 +[INFO] [1735952276.337183]: resetting environment +[INFO] [1735952276.342060]: cleared current plan +[INFO] [1735952276.343513]: moving to home +[INFO] [1735952277.346376]: reached home +[INFO] [1735952277.348518]: reset action history +[INFO] [1735952280.056795]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735952282.666167]: resetting environment +[INFO] [1735952282.669110]: cleared current plan +[INFO] [1735952282.669909]: moving to home +[INFO] [1735952283.672388]: reached home +[INFO] [1735952283.673820]: reset action history +[INFO] [1735952286.382340]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/episode_rosbags/episode_0_2025-01-03-19-58-06.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1960134.log +started roslaunch server http://158.130.50.37:34859/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1960411] +process[print_text-2]: started with pid [1960435] +process[rosbag_record-3]: started with pid [1960436] +started bagging! +For topic gripper_width: timestamp difference is 111105550 for nearest: 1735952311828291286 - target: 1735952311717185736 at idx 249 +[INFO] [1735952313.382180]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735952314.138100]: Stored episode 1. +[INFO] [1735952314.138590]: Saving video... +[INFO] [1735952314.324852]: Video saved! +Starting episode 1 +[INFO] [1735952314.355024]: resetting environment +[INFO] [1735952314.360180]: cleared current plan +[INFO] [1735952314.360421]: moving to home +[INFO] [1735952318.062890]: reached home +[INFO] [1735952318.063472]: reset action history +[INFO] [1735952320.770579]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735952324.590289]: resetting environment +[INFO] [1735952324.593773]: cleared current plan +[INFO] [1735952324.593932]: moving to home +[INFO] [1735952325.595319]: reached home +[INFO] [1735952325.595721]: reset action history +[INFO] [1735952328.301942]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/episode_rosbags/episode_1_2025-01-03-19-58-48.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1960134.log +started roslaunch server http://158.130.50.37:44789/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1960555] +process[print_text-5]: started with pid [1960556] +process[rosbag_record-6]: started with pid [1960557] +started bagging! +For topic gripper_width: timestamp difference is -203342435 for nearest: 1735952334961626294 - target: 1735952335164968729 at idx 92 +[INFO] [1735952355.265434]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1735952356.021868]: Stored episode 2. +[INFO] [1735952356.022499]: Saving video... +[INFO] [1735952356.509902]: Video saved! +Starting episode 2 +[INFO] [1735952356.545215]: resetting environment +[INFO] [1735952356.577594]: cleared current plan +[INFO] [1735952356.579494]: moving to home +[INFO] [1735952360.282750]: reached home +[INFO] [1735952360.291043]: reset action history +[INFO] [1735952363.059472]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735952367.031581]: resetting environment +[INFO] [1735952367.042264]: cleared current plan +[INFO] [1735952367.048813]: moving to home +[INFO] [1735952368.053530]: reached home +[INFO] [1735952368.054492]: reset action history +[INFO] [1735952370.766876]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/episode_rosbags/episode_2_2025-01-03-19-59-30.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1960134.log +started roslaunch server http://158.130.50.37:42815/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1960691] +process[print_text-8]: started with pid [1960692] +process[rosbag_record-9]: started with pid [1960716] +started bagging! +[INFO] [1735952397.803765]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1735952398.649749]: Stored episode 3. +[INFO] [1735952398.652726]: Saving video... +[INFO] [1735952399.331709]: Video saved! +Starting episode 3 +[INFO] [1735952399.387538]: resetting environment +[INFO] [1735952399.407208]: cleared current plan +[INFO] [1735952399.413205]: moving to home +[WARN] [1735952399.135222]: Plan exhausted +[WARN] [1735952399.174976]: Plan exhausted +[WARN] [1735952399.215051]: Plan exhausted +[WARN] [1735952399.255173]: Plan exhausted +[WARN] [1735952399.294914]: Plan exhausted +[WARN] [1735952399.337115]: Plan exhausted +[WARN] [1735952399.374943]: Plan exhausted +[INFO] [1735952403.229011]: reached home +[INFO] [1735952403.235353]: reset action history +[INFO] [1735952406.020122]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735952408.699650]: resetting environment +[INFO] [1735952408.726035]: cleared current plan +[INFO] [1735952408.731981]: moving to home +[INFO] [1735952409.740375]: reached home +[INFO] [1735952409.747040]: reset action history +[INFO] [1735952412.738507]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/episode_rosbags/episode_3_2025-01-03-20-00-12.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1960134.log +started roslaunch server http://158.130.50.37:34699/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1960829] +process[print_text-11]: started with pid [1960830] +process[rosbag_record-12]: started with pid [1960854] +started bagging! +For topic gripper_width: timestamp difference is -106477324 for nearest: 1735952413561623033 - target: 1735952413668100357 at idx 17 +For topic gripper_width: timestamp difference is -109683325 for nearest: 1735952422961650560 - target: 1735952423071333885 at idx 141 +[INFO] [1735952439.753249]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1735952440.523126]: Stored episode 4. +[INFO] [1735952440.530351]: Saving video... +[INFO] [1735952440.957241]: Video saved! +Starting episode 4 +[INFO] [1735952441.014637]: resetting environment +[INFO] [1735952441.037366]: cleared current plan +[INFO] [1735952441.039220]: moving to home +[INFO] [1735952444.852235]: reached home +[INFO] [1735952444.860827]: reset action history +[INFO] [1735952447.692688]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735952451.928995]: resetting environment +[INFO] [1735952451.943652]: cleared current plan +[INFO] [1735952451.954262]: moving to home +[INFO] [1735952452.959002]: reached home +[INFO] [1735952452.961742]: reset action history +[INFO] [1735952455.739200]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/episode_rosbags/episode_4_2025-01-03-20-00-55.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1960134.log +started roslaunch server http://158.130.50.37:35149/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1960997] +process[print_text-14]: started with pid [1960998] +process[rosbag_record-15]: started with pid [1960999] +started bagging! +For topic gripper_width: timestamp difference is -105288323 for nearest: 1735952458261639539 - target: 1735952458366927862 at idx 38 +For topic gripper_width: timestamp difference is 100300800 for nearest: 1735952461228289377 - target: 1735952461127988577 at idx 75 +[INFO] [1735952483.074321]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735952483.826852]: Stored episode 5. +[INFO] [1735952483.830044]: Saving video... +[INFO] [1735952484.484205]: Video saved! +[INFO] [1735952484.544373]: resetting environment +[INFO] [1735952484.572858]: cleared current plan +[INFO] [1735952484.575906]: moving to home +[WARN] [1735952484.435943]: Plan exhausted +[WARN] [1735952484.475926]: Plan exhausted +[WARN] [1735952484.515893]: Plan exhausted +[WARN] [1735952484.555887]: Plan exhausted +[INFO] [1735952488.390813]: reached home +[INFO] [1735952488.396032]: reset action history +[INFO] [1735952491.257936]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/requirements.txt b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-metadata.json b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..13f90d118639c3988f838c37d9b0a52020429dc8 --- /dev/null +++ b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T00:57:48.452488", + "startedAt": "2025-01-04T00:57:48.037416", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3936.315625, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4496.289, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 711.108470916748 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a7a5304166d32e4d503e7417e3460a7a7e858504 --- /dev/null +++ b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "242d6ca502a6c74dce290cd496f05a21c3f2e89c852b8b58e66ac52de0312c20", "size": 1133413, "path": "media/videos/eval/0_eval_0_242d6ca502a6c74dce29.mp4"}, "global_step": 665, "_timestamp": 1735952498.6323674, "_runtime": 230.5851993560791, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "92e676c219a8081d0b991ffddbce20df13b10f2d111101e9c90619203bf8ad86", "size": 1132108, "path": "media/videos/eval/1_eval_1_92e676c219a8081d0b99.mp4"}, "eval/num_success": 5.0, "episode": 10.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "95cc898a615ff7a378e0626fdf7a7061bedc3c968cba3ab46b0ec02a4d7f70b1", "size": 1138478, "path": "media/videos/eval/2_eval_2_95cc898a615ff7a378e0.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "ea9f563ddcabcb5060b655b365ba0570d2049fe2c325e2d60a77a5f182ca5781", "size": 1132960, "path": "media/videos/eval/3_eval_3_ea9f563ddcabcb5060b6.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "c9f7cad01419898a5561dea8588234d342a92c72fd87792dd7282118439bb0ed", "size": 1144971, "path": "media/videos/eval/4_eval_4_c9f7cad01419898a5561.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://j02lkomha07hdq4s0f5y5d2d33omyn3dpdwq9fvbdsn0y5hbsedjn6yu6gorg9ijx9d92u90j4xboohid5h3k3atso5vkw84bv3eiy1rjm3b7ido10uk4c9hw1z249ns/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://f50i0t0ovxgjcm3ekv4jmlsml7ntr8ol4jgnpbuapo9hitkz8qi9ijvyjsvoo5xzdvb7he1atvb8jhcuc4sod5bo6q88pig53y0jj0ihngtynm2p932s2raq4pjcisk4:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 10, "total_num_episodes": 10, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9892721175168917, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.7146403789090983, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.7492533623673705, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "802de47402cd97ff16985e7f9caeff0984676b3a683258550eeb2d6f43f4d0e7", "size": 420, "artifact_path": "wandb-client-artifact://4wt0l1y8fcxe6ufzbv0tyxoin6oosz1rb3yjsofacowkxcwey9qm2tvom6m3anp0tzd0g9zt49a09u8flx8ojrswg1y4onq1cb3glihfamejw1xbx4on4yfr1tnvpqb8/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://mqmyzffrptnv8yjpagdj4fqmx4lfqc46cyzkg192wgf8dicjfgs8j8ind45ze4uymumshmjte4cg6zqs5qywkdnpuj7h0kjw0zuxzucns5r0qjsle3hsv1p3hi1cvijr:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 231}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.9892721176147461, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.7146403789520264, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.7492533326148987, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/logs/debug-internal.log b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..99fdc0d6ee12d456f2b7cf6dd659a1af5b9ae33f --- /dev/null +++ b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/logs/debug-internal.log @@ -0,0 +1,716 @@ +2025-01-03 19:57:48,046 INFO StreamThr :1960299 [internal.py:wandb_internal():85] W&B internal server running at pid: 1960299, started at: 2025-01-03 19:57:48.046468 +2025-01-03 19:57:48,048 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status +2025-01-03 19:57:48,060 INFO WriterThread:1960299 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/run-k9rmk9dl.wandb +2025-01-03 19:57:48,060 DEBUG SenderThread:1960299 [sender.py:send():379] send: header +2025-01-03 19:57:48,067 DEBUG SenderThread:1960299 [sender.py:send():379] send: run +2025-01-03 19:57:48,343 INFO SenderThread:1960299 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files +2025-01-03 19:57:48,343 INFO SenderThread:1960299 [sender.py:_start_run_threads():1188] run started: k9rmk9dl with start time 1735952268.047168 +2025-01-03 19:57:48,357 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 19:57:48,357 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: check_version +2025-01-03 19:57:48,429 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 19:57:48,434 DEBUG HandlerThread:1960299 [system_info.py:__init__():26] System info init +2025-01-03 19:57:48,434 DEBUG HandlerThread:1960299 [system_info.py:__init__():41] System info init done +2025-01-03 19:57:48,434 INFO HandlerThread:1960299 [system_monitor.py:start():194] Starting system monitor +2025-01-03 19:57:48,434 INFO SystemMonitor:1960299 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 19:57:48,434 INFO HandlerThread:1960299 [system_monitor.py:probe():214] Collecting system info +2025-01-03 19:57:48,434 INFO SystemMonitor:1960299 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 19:57:48,435 INFO SystemMonitor:1960299 [interfaces.py:start():188] Started disk monitoring +2025-01-03 19:57:48,436 INFO SystemMonitor:1960299 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 19:57:48,436 INFO SystemMonitor:1960299 [interfaces.py:start():188] Started memory monitoring +2025-01-03 19:57:48,437 INFO SystemMonitor:1960299 [interfaces.py:start():188] Started network monitoring +2025-01-03 19:57:48,452 DEBUG HandlerThread:1960299 [system_info.py:probe():152] Probing system +2025-01-03 19:57:48,454 DEBUG HandlerThread:1960299 [system_info.py:_probe_git():137] Probing git +2025-01-03 19:57:48,459 DEBUG HandlerThread:1960299 [system_info.py:_probe_git():145] Probing git done +2025-01-03 19:57:48,459 DEBUG HandlerThread:1960299 [system_info.py:probe():200] Probing system done +2025-01-03 19:57:48,459 DEBUG HandlerThread:1960299 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T00:57:48.452488', 'startedAt': '2025-01-04T00:57:48.037416', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3936.315625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.289, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 711.108470916748}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 19:57:48,459 INFO HandlerThread:1960299 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 19:57:48,459 INFO HandlerThread:1960299 [system_monitor.py:probe():227] Publishing system info +2025-01-03 19:57:48,459 DEBUG HandlerThread:1960299 [system_info.py:_save_code():44] Saving code +2025-01-03 19:57:48,460 DEBUG HandlerThread:1960299 [system_info.py:_save_code():67] Saving code done +2025-01-03 19:57:48,460 DEBUG HandlerThread:1960299 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 19:57:48,524 DEBUG HandlerThread:1960299 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 19:57:48,524 INFO HandlerThread:1960299 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 19:57:48,528 DEBUG SenderThread:1960299 [sender.py:send():379] send: files +2025-01-03 19:57:48,528 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 19:57:48,528 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 19:57:48,528 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 19:57:48,763 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 19:57:48,763 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: python_packages +2025-01-03 19:57:48,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 19:57:48,764 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 19:57:48,765 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:48,766 INFO HandlerThread:1960299 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/tb" + save: true +} +control { + relay_id: "dfc456c70ff745de8fb44313dae7533c" +} +_info { + stream_id: "k9rmk9dl" +} + +2025-01-03 19:57:48,766 DEBUG HandlerThread:1960299 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 19:57:48,800 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/1mia9ah7-wandb-metadata.json +2025-01-03 19:57:48,806 INFO wandb-upload_2:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/2oqurbl6-diff.patch +2025-01-03 19:57:48,834 DEBUG SenderThread:1960299 [sender.py:send():379] send: telemetry +2025-01-03 19:57:48,835 DEBUG SenderThread:1960299 [sender.py:send():379] send: tbrecord +2025-01-03 19:57:48,835 DEBUG SenderThread:1960299 [sender.py:send():379] send: files +2025-01-03 19:57:48,835 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 with policy live +2025-01-03 19:57:48,836 INFO wandb-upload_1:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/9ev08r4b-code/FISH/eval_robot.py +2025-01-03 19:57:49,055 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/z46mxsa1-events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 19:57:49,345 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/diff.patch +2025-01-03 19:57:49,345 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 19:57:49,345 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-metadata.json +2025-01-03 19:57:49,345 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/requirements.txt +2025-01-03 19:57:49,345 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:57:49,346 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/code/FISH/eval_robot.py +2025-01-03 19:57:49,346 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/code +2025-01-03 19:57:49,346 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/code/FISH +2025-01-03 19:57:49,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:50,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:51,344 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:57:51,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:52,618 DEBUG SenderThread:1960299 [sender.py:send():379] send: config +2025-01-03 19:57:52,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:53,345 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:57:53,625 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:57:53,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:54,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:55,345 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:57:55,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:56,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:57,346 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:57:57,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:58,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:57:58,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:57:59,347 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:57:59,765 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:00,765 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:01,347 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:01,765 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:02,766 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:03,763 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 19:58:03,763 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 19:58:03,766 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:03,904 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:04,766 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:05,348 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:05,766 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:06,766 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:07,766 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:08,766 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:09,350 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:09,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:09,766 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:10,767 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:11,767 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:12,767 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:13,767 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:14,767 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:14,769 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:15,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:16,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:17,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:18,763 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 19:58:18,763 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 19:58:18,805 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:19,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:19,843 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:20,353 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/config.yaml +2025-01-03 19:58:20,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:21,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:22,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:23,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:24,771 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:25,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:25,771 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:26,771 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:27,771 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:28,772 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:29,772 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:30,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:30,772 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:31,772 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:32,772 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:33,365 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:33,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 19:58:33,764 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 19:58:33,772 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:34,347 DEBUG SenderThread:1960299 [sender.py:send():379] send: files +2025-01-03 19:58:34,347 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_242d6ca502a6c74dce29.mp4 with policy now +2025-01-03 19:58:34,353 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 19:58:34,354 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 19:58:34,354 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 19:58:34,354 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 19:58:34,366 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 19:58:34,366 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/0_eval_0_242d6ca502a6c74dce29.mp4 +2025-01-03 19:58:34,366 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media +2025-01-03 19:58:34,366 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos +2025-01-03 19:58:34,366 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval +2025-01-03 19:58:34,604 INFO wandb-upload_1:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/9v4xf6g9-media/videos/eval/0_eval_0_242d6ca502a6c74dce29.mp4 +2025-01-03 19:58:34,773 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:34,847 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/klc7xu2t-media/videos/eval/0_eval_0_242d6ca502a6c74dce29.mp4 +2025-01-03 19:58:35,366 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:35,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:36,361 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:36,773 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:37,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:38,773 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:39,367 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:39,773 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:40,813 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:41,774 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:41,814 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:42,774 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:43,369 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:43,774 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:44,774 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:45,369 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 19:58:45,774 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:46,775 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:47,370 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:47,637 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:47,774 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:48,437 DEBUG SystemMonitor:1960299 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 19:58:48,438 DEBUG SenderThread:1960299 [sender.py:send():379] send: stats +2025-01-03 19:58:48,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 19:58:48,764 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 19:58:48,775 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:49,371 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:49,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:50,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:51,372 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:51,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:52,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:52,897 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:53,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:54,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:55,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:56,776 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:57,374 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:58:57,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:58,322 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:58:58,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:58:59,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:00,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:01,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:02,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:03,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 19:59:03,764 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 19:59:03,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:03,823 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:04,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:05,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:06,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:07,777 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:08,778 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:09,772 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:09,778 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:10,778 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:11,778 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:12,778 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:13,778 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:14,779 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:15,266 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:15,779 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:16,536 DEBUG SenderThread:1960299 [sender.py:send():379] send: files +2025-01-03 19:59:16,536 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_92e676c219a8081d0b99.mp4 with policy now +2025-01-03 19:59:16,540 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 19:59:16,540 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 19:59:16,541 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 19:59:16,541 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 19:59:16,783 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:16,820 INFO wandb-upload_1:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/3ywqzj5d-media/videos/eval/1_eval_1_92e676c219a8081d0b99.mp4 +2025-01-03 19:59:17,380 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 19:59:17,380 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:59:17,380 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/1_eval_1_92e676c219a8081d0b99.mp4 +2025-01-03 19:59:17,380 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval +2025-01-03 19:59:17,656 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/jjudmfxq-media/videos/eval/1_eval_1_92e676c219a8081d0b99.mp4 +2025-01-03 19:59:17,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:18,438 DEBUG SenderThread:1960299 [sender.py:send():379] send: stats +2025-01-03 19:59:18,767 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 19:59:18,767 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 19:59:18,813 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:19,381 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:59:19,785 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:20,283 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:20,796 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:21,381 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:59:21,793 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:22,792 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:23,784 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:24,784 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:25,383 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:59:25,772 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:25,785 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:26,784 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:27,383 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 19:59:27,644 INFO wandb-upload_1:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/7s6ixnoj-events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 19:59:27,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:28,170 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 19:59:28,170 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 19:59:28,171 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 19:59:28,384 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 19:59:28,785 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:29,384 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:59:29,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:30,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:30,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:31,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:32,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:33,385 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:59:33,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 19:59:33,764 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 19:59:33,805 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:34,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:35,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:35,855 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:36,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:37,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:38,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:39,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:40,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:41,773 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:41,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:42,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:43,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:44,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:45,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:46,774 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:46,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:47,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:48,439 DEBUG SenderThread:1960299 [sender.py:send():379] send: stats +2025-01-03 19:59:48,764 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 19:59:48,764 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 19:59:48,805 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:49,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:50,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:51,788 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:51,836 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:52,788 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:53,788 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:54,788 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:55,788 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:56,788 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:57,774 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 19:59:57,788 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:58,793 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:59,384 DEBUG SenderThread:1960299 [sender.py:send():379] send: files +2025-01-03 19:59:59,384 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_95cc898a615ff7a378e0.mp4 with policy now +2025-01-03 19:59:59,387 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 19:59:59,387 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 19:59:59,388 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 19:59:59,388 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 19:59:59,394 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 19:59:59,394 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 19:59:59,394 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/2_eval_2_95cc898a615ff7a378e0.mp4 +2025-01-03 19:59:59,394 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval +2025-01-03 19:59:59,634 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/crmdl6ea-media/videos/eval/2_eval_2_95cc898a615ff7a378e0.mp4 +2025-01-03 19:59:59,797 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 19:59:59,874 INFO wandb-upload_1:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/zpt8gf3d-media/videos/eval/2_eval_2_95cc898a615ff7a378e0.mp4 +2025-01-03 20:00:00,800 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:01,394 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:01,804 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:02,799 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:03,231 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:03,770 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:00:03,771 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:00:03,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:04,802 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:05,396 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:05,808 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:06,808 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:07,397 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:07,806 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:08,665 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:08,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:09,002 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 20:00:09,002 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:00:09,003 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:00:09,397 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 20:00:09,398 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 20:00:09,818 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:10,810 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:11,398 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:11,814 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:12,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:13,753 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:13,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:14,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:15,399 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:15,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:16,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:17,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:18,439 DEBUG SenderThread:1960299 [sender.py:send():379] send: stats +2025-01-03 20:00:18,766 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:00:18,767 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:00:18,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:18,832 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:19,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:20,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:21,807 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:22,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:23,808 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:24,775 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:24,808 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:25,403 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:25,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:26,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:27,810 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:28,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:29,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:30,775 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:30,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:31,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:32,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:33,769 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:00:33,769 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:00:33,809 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:34,810 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:35,810 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:35,843 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:36,810 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:37,810 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:38,810 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:39,811 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:40,828 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:40,960 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:41,005 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:00:41,005 DEBUG SenderThread:1960299 [sender.py:send():379] send: files +2025-01-03 20:00:41,006 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ea9f563ddcabcb5060b6.mp4 with policy now +2025-01-03 20:00:41,006 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 20:00:41,006 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:00:41,007 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:00:41,408 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 20:00:41,408 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:41,408 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/3_eval_3_ea9f563ddcabcb5060b6.mp4 +2025-01-03 20:00:41,408 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval +2025-01-03 20:00:41,551 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/gjb4z54g-media/videos/eval/3_eval_3_ea9f563ddcabcb5060b6.mp4 +2025-01-03 20:00:41,793 INFO wandb-upload_1:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/axwvln0n-media/videos/eval/3_eval_3_ea9f563ddcabcb5060b6.mp4 +2025-01-03 20:00:41,826 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:42,826 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:43,409 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:43,828 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:44,822 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:45,835 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:46,834 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:46,893 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:47,410 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:47,821 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:48,439 DEBUG SenderThread:1960299 [sender.py:send():379] send: stats +2025-01-03 20:00:48,775 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:00:48,776 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:00:48,824 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:49,411 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:49,831 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:50,827 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:51,829 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:51,902 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:52,063 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 20:00:52,063 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:00:52,064 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:00:52,412 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 20:00:52,412 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 20:00:52,680 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/2oafbqye-events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 20:00:52,830 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:53,412 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:53,829 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:54,830 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:55,413 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:55,834 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:56,830 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:57,326 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:00:57,414 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:57,830 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:58,831 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:00:59,414 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:00:59,831 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:00,831 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:01,831 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:02,831 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:03,301 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:03,416 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:01:03,768 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:01:03,769 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:01:03,832 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:04,832 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:05,832 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:06,832 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:07,832 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:08,786 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:08,832 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:09,832 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:10,832 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:11,834 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:12,836 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:13,787 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:13,835 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:14,835 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:15,836 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:16,837 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:17,837 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:18,440 DEBUG SenderThread:1960299 [sender.py:send():379] send: stats +2025-01-03 20:01:18,768 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:01:18,769 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:01:18,837 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:18,843 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:19,837 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:20,837 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:21,837 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:22,838 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:23,869 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:23,869 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:24,538 DEBUG SenderThread:1960299 [sender.py:send():379] send: files +2025-01-03 20:01:24,538 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_c9f7cad01419898a5561.mp4 with policy now +2025-01-03 20:01:24,544 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:01:24,544 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 20:01:24,544 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:24,544 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:24,790 INFO wandb-upload_1:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/io18ifql-media/videos/eval/4_eval_4_c9f7cad01419898a5561.mp4 +2025-01-03 20:01:24,839 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:25,423 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 20:01:25,423 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:01:25,423 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/4_eval_4_c9f7cad01419898a5561.mp4 +2025-01-03 20:01:25,424 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval +2025-01-03 20:01:25,700 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/nowm3nn2-media/videos/eval/4_eval_4_c9f7cad01419898a5561.mp4 +2025-01-03 20:01:25,855 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:26,855 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:27,856 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:28,853 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:29,425 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:01:29,797 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:29,857 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:30,861 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:31,856 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:32,857 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:33,426 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:01:33,791 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:01:33,791 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:01:33,863 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:34,875 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:34,907 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:35,880 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:36,865 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:37,866 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:38,064 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:01:38,064 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:01:38,365 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:01:38,428 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 20:01:38,428 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 20:01:38,428 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media +2025-01-03 20:01:38,428 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table +2025-01-03 20:01:38,428 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table/eval +2025-01-03 20:01:38,536 INFO wandb-upload_1:1960299 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpqqw88j0_ +2025-01-03 20:01:38,631 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/5ok435hm-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 20:01:38,718 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:01:38,732 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:01:38,860 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:38,940 INFO SenderThread:1960299 [sender.py:send_request_log_artifact():1525] logged artifact run-k9rmk9dl-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzgxNjU4Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY2NjYwMg==', 'latestArtifact': None}} +2025-01-03 20:01:38,940 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 20:01:38,940 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,941 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,941 DEBUG SenderThread:1960299 [sender.py:send():379] send: files +2025-01-03 20:01:38,941 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-03 20:01:38,941 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 20:01:38,941 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,942 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,942 DEBUG SenderThread:1960299 [sender.py:send():379] send: telemetry +2025-01-03 20:01:38,942 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,942 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,942 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,943 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,943 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,945 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,945 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,945 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,945 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,946 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,946 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,946 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,946 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,947 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,947 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,947 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,947 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,948 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,948 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,948 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,948 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:38,949 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:38,949 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:01:39,203 INFO wandb-upload_2:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/wpw1wp1e-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 20:01:39,285 INFO wandb-upload_0:1960299 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpj1x3hdx1 +2025-01-03 20:01:39,428 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 20:01:39,428 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:01:39,428 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 20:01:39,429 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json +2025-01-03 20:01:39,429 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table/eval +2025-01-03 20:01:39,676 INFO SenderThread:1960299 [sender.py:send_request_log_artifact():1525] logged artifact run-k9rmk9dl-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzgxNjYwNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY2NjYwNQ==', 'latestArtifact': None}} +2025-01-03 20:01:39,676 DEBUG SenderThread:1960299 [sender.py:send():379] send: files +2025-01-03 20:01:39,676 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json with policy now +2025-01-03 20:01:39,677 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 20:01:39,677 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:39,677 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:39,853 INFO wandb-upload_1:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/d8h05qo4-media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json +2025-01-03 20:01:39,860 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:40,118 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/2q7wo8nj-media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json +2025-01-03 20:01:40,327 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:40,373 DEBUG SenderThread:1960299 [sender.py:send():379] send: exit +2025-01-03 20:01:40,373 INFO SenderThread:1960299 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 20:01:40,374 INFO SenderThread:1960299 [sender.py:send_exit():588] handling runtime: 231 +2025-01-03 20:01:40,374 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:40,374 INFO SenderThread:1960299 [sender.py:send_exit():594] send defer +2025-01-03 20:01:40,374 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:40,374 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 20:01:40,374 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:40,374 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 20:01:40,375 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 1 +2025-01-03 20:01:40,375 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:40,375 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 20:01:40,375 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:40,375 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 20:01:40,375 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 2 +2025-01-03 20:01:40,375 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:40,375 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 20:01:40,375 INFO HandlerThread:1960299 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 20:01:40,375 INFO HandlerThread:1960299 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 20:01:40,376 DEBUG SystemMonitor:1960299 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 20:01:40,376 DEBUG SystemMonitor:1960299 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 20:01:40,376 INFO HandlerThread:1960299 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 20:01:40,429 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 20:01:40,450 INFO HandlerThread:1960299 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 20:01:40,451 INFO HandlerThread:1960299 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 20:01:40,451 INFO HandlerThread:1960299 [interfaces.py:finish():200] Joined network monitor +2025-01-03 20:01:40,451 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:40,451 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 20:01:40,451 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 3 +2025-01-03 20:01:40,451 DEBUG SenderThread:1960299 [sender.py:send():379] send: stats +2025-01-03 20:01:40,452 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:40,452 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 20:01:40,452 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:40,452 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 20:01:40,452 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 4 +2025-01-03 20:01:40,452 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:40,452 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 20:01:41,429 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:01:47,126 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:01:47,127 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:47,127 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 20:01:47,128 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:47,128 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 20:01:47,128 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 5 +2025-01-03 20:01:47,128 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:01:47,128 DEBUG SenderThread:1960299 [sender.py:send():379] send: history +2025-01-03 20:01:47,128 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:01:47,128 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:47,129 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 20:01:47,129 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:47,129 DEBUG SenderThread:1960299 [sender.py:send():379] send: summary +2025-01-03 20:01:47,130 INFO SenderThread:1960299 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:01:47,130 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:47,130 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 20:01:47,130 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 6 +2025-01-03 20:01:47,130 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:47,130 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 20:01:47,130 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:47,130 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 20:01:47,134 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:01:47,239 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 7 +2025-01-03 20:01:47,240 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:47,240 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 20:01:47,240 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:47,240 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 20:01:47,375 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:01:47,432 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 20:01:47,432 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/config.yaml +2025-01-03 20:01:49,247 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 8 +2025-01-03 20:01:49,247 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:01:49,247 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:49,247 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 20:01:49,248 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:49,248 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 20:01:49,248 INFO SenderThread:1960299 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 20:01:49,250 INFO SenderThread:1960299 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 20:01:49,252 INFO SenderThread:1960299 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 20:01:49,260 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 9 +2025-01-03 20:01:49,260 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:49,260 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 20:01:49,260 DEBUG SenderThread:1960299 [sender.py:send():379] send: artifact +2025-01-03 20:01:49,376 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:01:49,433 INFO Thread-12 :1960299 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:01:49,466 INFO SenderThread:1960299 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzgxMTc2Nw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzgxMTc2Nw==', 'versionIndex': 405}}} +2025-01-03 20:01:49,466 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:49,466 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 20:01:49,466 INFO SenderThread:1960299 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 20:01:50,433 INFO SenderThread:1960299 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files +2025-01-03 20:01:50,434 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/config.yaml config.yaml +2025-01-03 20:01:50,434 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 20:01:50,434 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json wandb-summary.json +2025-01-03 20:01:50,434 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/requirements.txt requirements.txt +2025-01-03 20:01:50,434 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-metadata.json wandb-metadata.json +2025-01-03 20:01:50,434 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log output.log +2025-01-03 20:01:50,434 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/diff.patch diff.patch +2025-01-03 20:01:50,436 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 20:01:50,443 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/3_eval_3_ea9f563ddcabcb5060b6.mp4 media/videos/eval/3_eval_3_ea9f563ddcabcb5060b6.mp4 +2025-01-03 20:01:50,443 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/4_eval_4_c9f7cad01419898a5561.mp4 media/videos/eval/4_eval_4_c9f7cad01419898a5561.mp4 +2025-01-03 20:01:50,443 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/1_eval_1_92e676c219a8081d0b99.mp4 media/videos/eval/1_eval_1_92e676c219a8081d0b99.mp4 +2025-01-03 20:01:50,443 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/0_eval_0_242d6ca502a6c74dce29.mp4 media/videos/eval/0_eval_0_242d6ca502a6c74dce29.mp4 +2025-01-03 20:01:50,443 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/videos/eval/2_eval_2_95cc898a615ff7a378e0.mp4 media/videos/eval/2_eval_2_95cc898a615ff7a378e0.mp4 +2025-01-03 20:01:50,444 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-03 20:01:50,444 INFO SenderThread:1960299 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json +2025-01-03 20:01:50,444 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 10 +2025-01-03 20:01:50,444 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:01:50,445 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:50,445 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 20:01:50,445 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:50,445 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 20:01:50,445 INFO SenderThread:1960299 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:01:50,615 INFO wandb-upload_4:1960299 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/wandb-summary.json +2025-01-03 20:01:50,617 INFO wandb-upload_1:1960299 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/config.yaml +2025-01-03 20:01:50,670 INFO wandb-upload_0:1960299 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/requirements.txt +2025-01-03 20:01:50,683 INFO wandb-upload_2:1960299 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/files/output.log +2025-01-03 20:01:50,728 INFO wandb-upload_3:1960299 [upload_job.py:push():130] Uploaded file /tmp/tmp60hvhemswandb/cff81489-events.out.tfevents.1735952268.leonmkim-ROG-Strix-G15CS-G15CS.1960134.0 +2025-01-03 20:01:50,928 INFO Thread-11 (_thread_body):1960299 [sender.py:transition_state():614] send defer: 11 +2025-01-03 20:01:50,928 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:50,928 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 20:01:50,929 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:50,929 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 20:01:50,929 INFO SenderThread:1960299 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 20:01:50,930 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 12 +2025-01-03 20:01:50,930 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:50,930 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 20:01:50,930 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:50,930 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 20:01:50,930 INFO SenderThread:1960299 [file_stream.py:finish():601] file stream finish called +2025-01-03 20:01:51,006 INFO SenderThread:1960299 [file_stream.py:finish():605] file stream finish is done +2025-01-03 20:01:51,006 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 13 +2025-01-03 20:01:51,006 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:51,006 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 20:01:51,006 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:51,006 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 20:01:51,007 INFO SenderThread:1960299 [sender.py:transition_state():614] send defer: 14 +2025-01-03 20:01:51,007 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:01:51,007 INFO HandlerThread:1960299 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 20:01:51,007 DEBUG SenderThread:1960299 [sender.py:send():379] send: final +2025-01-03 20:01:51,007 DEBUG SenderThread:1960299 [sender.py:send():379] send: footer +2025-01-03 20:01:51,007 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: defer +2025-01-03 20:01:51,007 INFO SenderThread:1960299 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 20:01:51,008 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:01:51,008 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 20:01:51,008 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:01:51,008 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:01:51,008 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 20:01:51,009 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: server_info +2025-01-03 20:01:51,009 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 20:01:51,011 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:01:51,069 DEBUG SenderThread:1960299 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:01:51,069 INFO MainThread:1960299 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 20:01:51,069 INFO MainThread:1960299 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 20:01:51,070 INFO MainThread:1960299 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 20:01:51,070 DEBUG HandlerThread:1960299 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 20:01:51,070 INFO HandlerThread:1960299 [handler.py:finish():882] shutting down handler +2025-01-03 20:01:52,008 INFO WriterThread:1960299 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/run-k9rmk9dl.wandb +2025-01-03 20:01:52,069 INFO SenderThread:1960299 [sender.py:finish():1615] shutting down sender +2025-01-03 20:01:52,069 INFO SenderThread:1960299 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:01:52,069 INFO SenderThread:1960299 [file_pusher.py:join():175] waiting for file pusher diff --git a/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/logs/debug.log b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bff87eabeed784a8986e4f6485740c01c29716f0 --- /dev/null +++ b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 19:57:48,040 INFO MainThread:1960134 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 19:57:48,040 INFO MainThread:1960134 [wandb_setup.py:_flush():76] Configure stats pid to 1960134 +2025-01-03 19:57:48,040 INFO MainThread:1960134 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/settings +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/logs/debug.log +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/logs/debug-internal.log +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_init.py:init():569] calling init triggers +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl'} +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_init.py:init():619] starting backend +2025-01-03 19:57:48,041 INFO MainThread:1960134 [wandb_init.py:init():623] setting up manager +2025-01-03 19:57:48,045 INFO MainThread:1960134 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 19:57:48,047 INFO MainThread:1960134 [wandb_init.py:init():631] backend started and connected +2025-01-03 19:57:48,059 INFO MainThread:1960134 [wandb_init.py:init():720] updated telemetry +2025-01-03 19:57:48,064 INFO MainThread:1960134 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 19:57:48,357 INFO MainThread:1960134 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 19:57:48,423 INFO MainThread:1960134 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 19:57:48,424 INFO MainThread:1960134 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 19:57:48,763 INFO MainThread:1960134 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 19:57:48,764 INFO MainThread:1960134 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 19:57:48,764 INFO MainThread:1960134 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 19:57:48,764 INFO MainThread:1960134 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 19:57:48,765 INFO MainThread:1960134 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 19:57:48,765 INFO MainThread:1960134 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/k9rmk9dl/tb, True +2025-01-03 19:57:52,617 INFO MainThread:1960134 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-03 20:01:52,072 WARNING MsgRouterThr:1960134 [router.py:message_loop():77] message_loop has been closed diff --git a/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/run-k9rmk9dl.wandb b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/run-k9rmk9dl.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e7da56722e264d175ab90291c91fb530ad2c93df Binary files /dev/null and b/k9rmk9dl/wandb/run-20250103_195748-k9rmk9dl/run-k9rmk9dl.wandb differ diff --git a/ljrlz21b/.hydra/config.yaml b/ljrlz21b/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50c4489cc892923b6fde889d218eeedfe242c7ee --- /dev/null +++ b/ljrlz21b/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1922_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/ljrlz21b/.hydra/hydra.yaml b/ljrlz21b/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca93b1f110ddb802c161f340c90e63100dfb9718 --- /dev/null +++ b/ljrlz21b/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/ljrlz21b + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/ljrlz21b/.hydra/overrides.yaml b/ljrlz21b/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/ljrlz21b/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/ljrlz21b/eval_policy.log b/ljrlz21b/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..6591c7d06c9649c5e8d39fb9583da961c4c32522 --- /dev/null +++ b/ljrlz21b/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-03 22:16:02,874][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 22:16:02,878][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-03 22:16:06,916][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/nhx7r1z0/.hydra/config.yaml b/nhx7r1z0/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50c4489cc892923b6fde889d218eeedfe242c7ee --- /dev/null +++ b/nhx7r1z0/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1922_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/nhx7r1z0/.hydra/hydra.yaml b/nhx7r1z0/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5c9ea1f0f72dc17308b9f9799952ac6aa5e53eb --- /dev/null +++ b/nhx7r1z0/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/nhx7r1z0/.hydra/overrides.yaml b/nhx7r1z0/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/nhx7r1z0/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/nhx7r1z0/episode_rosbags/episode_0_2025-01-03-22-28-51.bag b/nhx7r1z0/episode_rosbags/episode_0_2025-01-03-22-28-51.bag new file mode 100644 index 0000000000000000000000000000000000000000..a655a42c0b6ff14293bdef442b0e8204a088fd5b --- /dev/null +++ b/nhx7r1z0/episode_rosbags/episode_0_2025-01-03-22-28-51.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fc74c5551d3febe192ec3a27302fdf7c6003874e9370277ef1d589d8687136d +size 1410813979 diff --git a/nhx7r1z0/episode_rosbags/episode_1_2025-01-03-22-30-21.bag b/nhx7r1z0/episode_rosbags/episode_1_2025-01-03-22-30-21.bag new file mode 100644 index 0000000000000000000000000000000000000000..e6b458768f38d8c2a8e422eac20f4ecf4abf9a49 --- /dev/null +++ b/nhx7r1z0/episode_rosbags/episode_1_2025-01-03-22-30-21.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d48b928fab0582c904e66a581ff2154f8c59fbc6d341186bfd452c501a84ddf +size 1410898604 diff --git a/nhx7r1z0/episode_rosbags/episode_2_2025-01-03-22-31-04.bag b/nhx7r1z0/episode_rosbags/episode_2_2025-01-03-22-31-04.bag new file mode 100644 index 0000000000000000000000000000000000000000..e8bb74117e84df7110080926b935bc1445ee927e --- /dev/null +++ b/nhx7r1z0/episode_rosbags/episode_2_2025-01-03-22-31-04.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a20ac4f92ff6af957e9cdcc2adab3902f52469718dea98e5ef493ce4d2511ae +size 1412647887 diff --git a/nhx7r1z0/episode_rosbags/episode_3_2025-01-03-22-32-09.bag b/nhx7r1z0/episode_rosbags/episode_3_2025-01-03-22-32-09.bag new file mode 100644 index 0000000000000000000000000000000000000000..34bec8a5ad09f95e5025974870db18bd8ede6af5 --- /dev/null +++ b/nhx7r1z0/episode_rosbags/episode_3_2025-01-03-22-32-09.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51838e5ab4620c26e3063464c81fbde21661783f82c34e411286a76a0dd24dc9 +size 1288593909 diff --git a/nhx7r1z0/episode_rosbags/episode_4_2025-01-03-22-32-57.bag b/nhx7r1z0/episode_rosbags/episode_4_2025-01-03-22-32-57.bag new file mode 100644 index 0000000000000000000000000000000000000000..5816c1a6f4f7c8c4818f715c6a1466de77fd9665 --- /dev/null +++ b/nhx7r1z0/episode_rosbags/episode_4_2025-01-03-22-32-57.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c24a1053f9764da1e633957d2bf99244814bbf1b91fa3b556b8492ae757cc9dd +size 1412255753 diff --git a/nhx7r1z0/eval_video/0_eval.mp4 b/nhx7r1z0/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..05c0491305ee243b1ec5de6d865d4e6638fcf109 --- /dev/null +++ b/nhx7r1z0/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b79de18ba2122e2eecafa013d1c65c981cc55bc451baebaa06c8943c65a868f +size 1211156 diff --git a/nhx7r1z0/eval_video/1_eval.mp4 b/nhx7r1z0/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..121427111a9894ee2242ad933c43e88d7eae346b --- /dev/null +++ b/nhx7r1z0/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5d85a1429df5e5a3e60b72af66dc6f95d17129210f24776fc22b6a8be7ce7db +size 1175549 diff --git a/nhx7r1z0/eval_video/2_eval.mp4 b/nhx7r1z0/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c5fe8b51688c1a7efbee30e6fd8c363bee759fa5 --- /dev/null +++ b/nhx7r1z0/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3966ddb8583e8b4d202c0e9183dc73074df0e024050b9367b17669b1818d8767 +size 1161759 diff --git a/nhx7r1z0/eval_video/3_eval.mp4 b/nhx7r1z0/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f80a283d31ac54c22efb9a41deefb21ade345a0 --- /dev/null +++ b/nhx7r1z0/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c408001aca8abf0c7ef9304a8f8dbd796c01659db8c6e2de22e55aaf8019fe4a +size 1144006 diff --git a/nhx7r1z0/eval_video/4_eval.mp4 b/nhx7r1z0/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ff098a91aff1650a42c6511679692e6482a2f2cd --- /dev/null +++ b/nhx7r1z0/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d93a45e1282e6c91c0f9e54b9ae944c5239c7467e2b08b9b2fc2066d7862812c +size 1230887 diff --git a/nhx7r1z0/tb/events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 b/nhx7r1z0/tb/events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 new file mode 100644 index 0000000000000000000000000000000000000000..b7fc38e899b9d8edf6540388d59f4012eeed437d --- /dev/null +++ b/nhx7r1z0/tb/events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5911fa1f2ee7de1829b39634a8168b4fa97b41c8f6bbdc8aaea4a2565d95bc25 +size 6539 diff --git a/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/code/FISH/eval_robot.py b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json new file mode 100644 index 0000000000000000000000000000000000000000..d1a9a8f5261253760795e8c0f15e63e65e093e60 --- /dev/null +++ b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.4, 0.09420213111410328, 0.8653537630304143, 0.08973243840793546, 0.8341690897273251, 0.12645963586887082, 0.7050612568445163, 0.11327369891391026, 0.6491125076710976]]} \ No newline at end of file diff --git a/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json new file mode 100644 index 0000000000000000000000000000000000000000..9d792d4006105450f6870c1e1f2a1e18fb615772 --- /dev/null +++ b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[22, 25, 0.88, 0.7029817103933635, 0.9734671934492175, 0.6976236402593188, 0.9621387421160531, 0.7217742651257426, 0.9598798117611944, 0.7949399984783139, 0.9499553627365946]]} \ No newline at end of file diff --git a/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/0_eval_0_3b79de18ba2122e2eeca.mp4 b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/0_eval_0_3b79de18ba2122e2eeca.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..05c0491305ee243b1ec5de6d865d4e6638fcf109 --- /dev/null +++ b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/0_eval_0_3b79de18ba2122e2eeca.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b79de18ba2122e2eecafa013d1c65c981cc55bc451baebaa06c8943c65a868f +size 1211156 diff --git a/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/1_eval_1_f5d85a1429df5e5a3e60.mp4 b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/1_eval_1_f5d85a1429df5e5a3e60.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..121427111a9894ee2242ad933c43e88d7eae346b --- /dev/null +++ b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/1_eval_1_f5d85a1429df5e5a3e60.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5d85a1429df5e5a3e60b72af66dc6f95d17129210f24776fc22b6a8be7ce7db +size 1175549 diff --git a/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/2_eval_2_3966ddb8583e8b4d202c.mp4 b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/2_eval_2_3966ddb8583e8b4d202c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c5fe8b51688c1a7efbee30e6fd8c363bee759fa5 --- /dev/null +++ b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/2_eval_2_3966ddb8583e8b4d202c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3966ddb8583e8b4d202c0e9183dc73074df0e024050b9367b17669b1818d8767 +size 1161759 diff --git a/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/3_eval_3_c408001aca8abf0c7ef9.mp4 b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/3_eval_3_c408001aca8abf0c7ef9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f80a283d31ac54c22efb9a41deefb21ade345a0 --- /dev/null +++ b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/3_eval_3_c408001aca8abf0c7ef9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c408001aca8abf0c7ef9304a8f8dbd796c01659db8c6e2de22e55aaf8019fe4a +size 1144006 diff --git a/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/4_eval_4_d93a45e1282e6c91c0f9.mp4 b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/4_eval_4_d93a45e1282e6c91c0f9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ff098a91aff1650a42c6511679692e6482a2f2cd --- /dev/null +++ b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/4_eval_4_d93a45e1282e6c91c0f9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d93a45e1282e6c91c0f9e54b9ae944c5239c7467e2b08b9b2fc2066d7862812c +size 1230887 diff --git a/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/logs/debug-internal.log b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..df43b1c3630bdd874b61feff00962755c9ed3bc4 --- /dev/null +++ b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/logs/debug-internal.log @@ -0,0 +1,869 @@ +2025-01-03 22:28:18,557 INFO StreamThr :1983623 [internal.py:wandb_internal():85] W&B internal server running at pid: 1983623, started at: 2025-01-03 22:28:18.556934 +2025-01-03 22:28:18,559 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:28:18,571 INFO WriterThread:1983623 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/run-nhx7r1z0.wandb +2025-01-03 22:28:18,571 DEBUG SenderThread:1983623 [sender.py:send():379] send: header +2025-01-03 22:28:18,586 DEBUG SenderThread:1983623 [sender.py:send():379] send: run +2025-01-03 22:28:18,805 INFO SenderThread:1983623 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files +2025-01-03 22:28:18,805 INFO SenderThread:1983623 [sender.py:_start_run_threads():1188] run started: nhx7r1z0 with start time 1735961298.557627 +2025-01-03 22:28:18,829 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:28:18,830 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:28:18,906 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:28:18,911 DEBUG HandlerThread:1983623 [system_info.py:__init__():26] System info init +2025-01-03 22:28:18,911 DEBUG HandlerThread:1983623 [system_info.py:__init__():41] System info init done +2025-01-03 22:28:18,911 INFO HandlerThread:1983623 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:28:18,911 INFO SystemMonitor:1983623 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:28:18,911 INFO HandlerThread:1983623 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:28:18,912 INFO SystemMonitor:1983623 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:28:18,912 INFO SystemMonitor:1983623 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:28:18,913 INFO SystemMonitor:1983623 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:28:18,913 INFO SystemMonitor:1983623 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:28:18,914 INFO SystemMonitor:1983623 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:28:18,924 DEBUG HandlerThread:1983623 [system_info.py:probe():152] Probing system +2025-01-03 22:28:18,925 DEBUG HandlerThread:1983623 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:28:18,930 DEBUG HandlerThread:1983623 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:28:18,930 DEBUG HandlerThread:1983623 [system_info.py:probe():200] Probing system done +2025-01-03 22:28:18,930 DEBUG HandlerThread:1983623 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:28:18.924324', 'startedAt': '2025-01-04T03:28:18.544196', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.393, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.982, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.924, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 844.8166198730469}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:28:18,930 INFO HandlerThread:1983623 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:28:18,930 INFO HandlerThread:1983623 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:28:18,930 DEBUG HandlerThread:1983623 [system_info.py:_save_code():44] Saving code +2025-01-03 22:28:18,932 DEBUG HandlerThread:1983623 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:28:18,932 DEBUG HandlerThread:1983623 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:28:18,993 DEBUG HandlerThread:1983623 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:28:18,993 INFO HandlerThread:1983623 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:28:18,997 DEBUG SenderThread:1983623 [sender.py:send():379] send: files +2025-01-03 22:28:18,997 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:28:18,997 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:28:18,997 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:28:19,229 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:28:19,229 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:28:19,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:28:19,230 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:28:19,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:19,232 INFO HandlerThread:1983623 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/tb" + save: true +} +control { + relay_id: "513ce0d58ed945ebb8be16b792d16108" +} +_info { + stream_id: "nhx7r1z0" +} + +2025-01-03 22:28:19,232 DEBUG HandlerThread:1983623 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:28:19,244 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/uah3b2wt-wandb-metadata.json +2025-01-03 22:28:19,278 INFO wandb-upload_2:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/nsdn662v-diff.patch +2025-01-03 22:28:19,329 INFO wandb-upload_1:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/4a6tmd9s-code/FISH/eval_robot.py +2025-01-03 22:28:19,353 DEBUG SenderThread:1983623 [sender.py:send():379] send: telemetry +2025-01-03 22:28:19,353 DEBUG SenderThread:1983623 [sender.py:send():379] send: tbrecord +2025-01-03 22:28:19,353 DEBUG SenderThread:1983623 [sender.py:send():379] send: files +2025-01-03 22:28:19,353 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 with policy live +2025-01-03 22:28:19,543 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/56emz1jk-events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:28:19,806 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/code/FISH/eval_robot.py +2025-01-03 22:28:19,807 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/diff.patch +2025-01-03 22:28:19,807 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-metadata.json +2025-01-03 22:28:19,807 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:28:19,807 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/requirements.txt +2025-01-03 22:28:19,807 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:28:19,807 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/code +2025-01-03 22:28:19,807 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/code/FISH +2025-01-03 22:28:20,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:21,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:21,806 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:28:22,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:23,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:24,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:28:24,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:24,995 DEBUG SenderThread:1983623 [sender.py:send():379] send: config +2025-01-03 22:28:25,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:26,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:27,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:27,807 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:28:28,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:29,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:28:29,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:29,808 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:28:30,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:31,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:31,809 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:28:32,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:33,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:33,809 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:28:34,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:28:34,230 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:28:34,273 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:34,384 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:28:35,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:36,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:37,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:38,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:39,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:40,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:28:40,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:41,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:42,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:43,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:44,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:45,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:46,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:28:46,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:47,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:48,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:49,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:28:49,230 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:28:49,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:49,814 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:28:50,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:51,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:51,338 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:28:51,814 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/config.yaml +2025-01-03 22:28:51,814 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:28:52,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:53,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:53,815 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:28:54,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:55,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:56,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:57,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:28:57,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:58,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:28:59,237 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:00,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:01,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:02,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:03,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:03,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:04,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:29:04,230 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:29:04,277 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:05,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:06,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:07,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:08,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:08,374 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:09,236 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:09,819 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:29:10,236 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:11,236 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:12,237 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:13,237 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:14,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:14,237 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:15,238 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:16,238 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:17,238 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:18,238 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:18,914 DEBUG SystemMonitor:1983623 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:29:18,915 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:29:19,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:29:19,230 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:29:19,238 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:19,380 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:19,635 DEBUG SenderThread:1983623 [sender.py:send():379] send: files +2025-01-03 22:29:19,635 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:29:19,635 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_3b79de18ba2122e2eeca.mp4 with policy now +2025-01-03 22:29:19,636 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:29:19,637 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:29:19,637 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:29:19,822 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:29:19,823 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:29:19,823 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/0_eval_0_3b79de18ba2122e2eeca.mp4 +2025-01-03 22:29:19,823 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media +2025-01-03 22:29:19,823 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval +2025-01-03 22:29:19,823 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos +2025-01-03 22:29:19,923 INFO wandb-upload_1:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/ux2n87nt-media/videos/eval/0_eval_0_3b79de18ba2122e2eeca.mp4 +2025-01-03 22:29:20,197 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/y8opkeng-media/videos/eval/0_eval_0_3b79de18ba2122e2eeca.mp4 +2025-01-03 22:29:20,239 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:21,240 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:21,823 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:29:22,239 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:23,239 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:24,244 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:25,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:25,239 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:25,824 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:29:26,240 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:27,240 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:27,825 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:29:28,240 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:29,240 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:30,240 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:31,232 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:31,240 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:32,241 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:33,241 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:34,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:29:34,230 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:29:34,277 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:35,241 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:36,241 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:36,364 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:37,241 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:38,241 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:39,241 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:40,241 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:41,242 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:42,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:42,242 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:43,242 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:44,242 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:45,242 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:46,242 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:47,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:47,242 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:48,242 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:48,915 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:29:49,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:29:49,230 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:29:49,273 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:50,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:51,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:52,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:52,382 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:53,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:54,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:55,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:56,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:57,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:58,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:29:58,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:29:59,244 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:00,244 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:01,244 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:02,244 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:03,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:03,244 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:04,230 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:30:04,231 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:30:04,273 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:05,244 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:06,244 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:07,244 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:08,245 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:08,313 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:09,245 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:10,245 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:11,245 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:12,245 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:13,245 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:14,234 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:14,245 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:15,245 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:16,246 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:17,246 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:17,843 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:30:18,087 INFO wandb-upload_1:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/keyst1n2-events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:30:18,246 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:18,916 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:30:19,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:30:19,231 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:30:19,246 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:19,347 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:19,844 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:30:20,246 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:21,265 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:22,246 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:23,246 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:23,845 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:30:24,247 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:25,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:25,247 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:26,247 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:27,247 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:28,247 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:29,247 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:30,247 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:31,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:31,248 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:32,248 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:33,248 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:34,231 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:30:34,231 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:30:34,273 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:35,248 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:36,248 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:36,388 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:37,248 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:38,248 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:39,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:40,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:41,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:42,235 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:42,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:43,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:44,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:45,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:46,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:47,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:48,031 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:48,253 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:48,916 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:30:49,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:30:49,233 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:30:49,254 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:49,688 DEBUG SenderThread:1983623 [sender.py:send():379] send: files +2025-01-03 22:30:49,688 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f5d85a1429df5e5a3e60.mp4 with policy now +2025-01-03 22:30:49,689 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:30:49,689 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:30:49,690 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:30:49,693 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:30:49,856 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:30:49,856 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:30:49,856 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/1_eval_1_f5d85a1429df5e5a3e60.mp4 +2025-01-03 22:30:49,856 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval +2025-01-03 22:30:49,981 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/nu95xmxi-media/videos/eval/1_eval_1_f5d85a1429df5e5a3e60.mp4 +2025-01-03 22:30:50,241 INFO wandb-upload_1:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/q85jsv4e-media/videos/eval/1_eval_1_f5d85a1429df5e5a3e60.mp4 +2025-01-03 22:30:50,255 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:51,254 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:51,857 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:30:52,261 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:53,255 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:53,315 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:54,258 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:55,264 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:55,858 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:30:56,261 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:57,261 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:57,859 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:30:58,261 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:30:59,133 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:30:59,261 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:00,261 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:00,695 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:31:00,695 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:31:00,696 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:31:00,860 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:31:00,860 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:31:01,261 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:01,860 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:31:02,261 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:03,261 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:03,861 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:31:04,209 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:04,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:31:04,249 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:31:04,264 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:05,262 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:05,862 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:31:06,262 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:07,262 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:08,262 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:09,236 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:09,262 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:10,262 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:11,263 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:12,263 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:13,263 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:13,864 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:31:14,236 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:14,263 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:15,264 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:16,264 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:17,264 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:18,264 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:18,916 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:31:19,233 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:31:19,233 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:31:19,281 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:19,380 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:20,265 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:21,265 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:22,265 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:23,265 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:24,265 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:25,236 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:25,265 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:26,265 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:27,266 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:28,266 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:29,266 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:30,237 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:30,266 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:31,266 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:32,268 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:32,394 DEBUG SenderThread:1983623 [sender.py:send():379] send: files +2025-01-03 22:31:32,394 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_3966ddb8583e8b4d202c.mp4 with policy now +2025-01-03 22:31:32,396 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:31:32,397 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:31:32,397 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:31:32,397 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:31:32,648 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/bvxodu4k-media/videos/eval/2_eval_2_3966ddb8583e8b4d202c.mp4 +2025-01-03 22:31:32,871 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:31:32,871 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/2_eval_2_3966ddb8583e8b4d202c.mp4 +2025-01-03 22:31:32,871 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval +2025-01-03 22:31:33,145 INFO wandb-upload_1:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/4wlgxdx7-media/videos/eval/2_eval_2_3966ddb8583e8b4d202c.mp4 +2025-01-03 22:31:33,270 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:33,871 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:31:34,240 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:31:34,240 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:31:34,281 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:35,272 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:35,327 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:36,275 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:37,278 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:37,872 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:31:38,277 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:39,281 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:39,873 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:31:40,276 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:41,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:41,279 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:42,281 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:43,280 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:44,280 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:45,281 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:46,246 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:46,284 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:47,282 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:48,288 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:48,917 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:31:49,237 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:31:49,237 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:31:49,287 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:50,291 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:51,286 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:51,338 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:52,287 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:53,289 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:54,292 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:55,291 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:56,288 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:57,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:31:57,391 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:58,387 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:31:59,388 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:00,389 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:01,388 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:02,250 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:02,390 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:03,390 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:04,238 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:32:04,238 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:32:04,393 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:05,393 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:05,561 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:32:05,561 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:32:05,561 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:32:05,881 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:32:05,882 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:32:06,116 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/9e0wntl7-events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:32:06,394 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:07,390 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:07,394 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:07,882 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:32:08,404 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:09,404 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:09,883 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:32:10,404 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:11,405 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:11,884 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:32:12,409 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:13,331 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:13,409 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:14,409 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:15,409 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:16,409 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:17,410 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:18,415 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:18,917 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:32:18,918 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:19,236 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:32:19,236 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:32:19,415 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:20,415 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:21,415 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:22,415 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:23,415 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:24,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:24,415 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:25,416 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:26,416 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:27,416 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:28,416 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:29,249 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:29,416 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:30,417 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:31,417 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:31,891 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:32:32,417 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:33,417 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:34,239 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:32:34,239 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:32:34,374 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:34,425 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:34,977 DEBUG SenderThread:1983623 [sender.py:send():379] send: files +2025-01-03 22:32:34,977 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c408001aca8abf0c7ef9.mp4 with policy now +2025-01-03 22:32:34,978 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:32:34,978 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:32:34,978 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:32:34,979 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:32:35,229 INFO wandb-upload_1:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/y1m9gucn-media/videos/eval/3_eval_3_c408001aca8abf0c7ef9.mp4 +2025-01-03 22:32:35,425 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:35,892 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:32:35,892 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:32:35,892 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/3_eval_3_c408001aca8abf0c7ef9.mp4 +2025-01-03 22:32:35,892 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval +2025-01-03 22:32:36,174 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/wazreo3l-media/videos/eval/3_eval_3_c408001aca8abf0c7ef9.mp4 +2025-01-03 22:32:36,426 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:37,427 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:38,430 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:39,433 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:39,893 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:32:40,266 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:40,437 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:41,424 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:42,433 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:43,434 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:43,895 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:32:44,428 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:45,438 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:46,251 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:46,436 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:47,435 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:48,435 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:48,918 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:32:49,243 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:32:49,244 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:32:49,431 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:50,437 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:51,298 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:51,437 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:52,433 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:53,440 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:53,619 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:32:53,620 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:32:53,621 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:32:53,898 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:32:53,898 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:32:54,465 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:55,443 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:55,899 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:32:56,446 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:56,466 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:32:57,450 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:58,449 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:59,445 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:32:59,900 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:00,445 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:01,445 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:02,253 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:02,445 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:03,473 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:04,237 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:33:04,237 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:33:04,445 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:05,445 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:05,902 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:06,446 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:07,446 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:07,505 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:08,446 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:09,446 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:10,446 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:11,447 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:12,447 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:13,305 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:13,447 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:13,905 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:14,447 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:15,447 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:16,447 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:17,448 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:18,439 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:18,481 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:18,918 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:33:19,237 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:33:19,238 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:33:19,448 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:19,907 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:20,448 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:21,448 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:21,908 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:22,448 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:23,448 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:24,253 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:24,448 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:25,448 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:26,454 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:27,096 DEBUG SenderThread:1983623 [sender.py:send():379] send: files +2025-01-03 22:33:27,096 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d93a45e1282e6c91c0f9.mp4 with policy now +2025-01-03 22:33:27,100 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:33:27,106 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:33:27,106 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:27,106 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:27,358 INFO wandb-upload_1:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/xzlexb6d-media/videos/eval/4_eval_4_d93a45e1282e6c91c0f9.mp4 +2025-01-03 22:33:27,454 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:27,910 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:27,911 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:33:27,911 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/4_eval_4_d93a45e1282e6c91c0f9.mp4 +2025-01-03 22:33:27,911 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval +2025-01-03 22:33:28,188 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/66kwga2b-media/videos/eval/4_eval_4_d93a45e1282e6c91c0f9.mp4 +2025-01-03 22:33:28,470 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:29,257 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:29,472 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:30,469 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:31,474 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:31,912 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:32,473 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:33,475 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:34,251 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:33:34,251 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:33:34,472 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:34,607 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:35,490 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:35,913 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:36,487 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:37,482 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:38,485 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:39,486 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:40,268 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:40,487 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:41,491 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:42,495 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:43,490 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:44,495 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:45,492 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:46,263 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:46,488 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:47,490 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:48,488 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:48,919 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:33:49,247 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:33:49,248 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:33:49,495 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:50,494 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:51,506 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:51,607 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:33:51,608 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:33:51,893 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:33:51,919 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:51,919 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:33:51,919 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json +2025-01-03 22:33:51,920 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media +2025-01-03 22:33:51,920 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table/eval +2025-01-03 22:33:51,920 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table +2025-01-03 22:33:52,094 INFO wandb-upload_2:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/yycw34ng-media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json +2025-01-03 22:33:52,122 INFO wandb-upload_1:1983623 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp99ywx_oj +2025-01-03 22:33:52,184 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/3klijqxb-events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:33:52,282 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:33:52,291 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:33:52,489 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:52,523 INFO SenderThread:1983623 [sender.py:send_request_log_artifact():1525] logged artifact run-nhx7r1z0-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk3OTgyNg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4ODU1OQ==', 'latestArtifact': None}} +2025-01-03 22:33:52,524 DEBUG SenderThread:1983623 [sender.py:send():379] send: files +2025-01-03 22:33:52,524 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json with policy now +2025-01-03 22:33:52,524 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:52,524 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:33:52,524 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,525 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,525 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:33:52,525 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,529 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,529 DEBUG SenderThread:1983623 [sender.py:send():379] send: telemetry +2025-01-03 22:33:52,529 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,530 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,530 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,530 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,530 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,530 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,531 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,531 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,531 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,531 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,532 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,532 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,532 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,533 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,533 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,533 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,533 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,534 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,534 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,534 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,534 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:52,535 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:52,535 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:33:52,783 INFO wandb-upload_2:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/3jtopk98-media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json +2025-01-03 22:33:52,919 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:33:52,920 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:33:52,920 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json +2025-01-03 22:33:52,920 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table/eval +2025-01-03 22:33:52,949 INFO wandb-upload_0:1983623 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9j7x7ffg +2025-01-03 22:33:53,104 INFO wandb-upload_1:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/57uppe4r-media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json +2025-01-03 22:33:53,439 INFO SenderThread:1983623 [sender.py:send_request_log_artifact():1525] logged artifact run-nhx7r1z0-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk3OTgzNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4ODU2MQ==', 'latestArtifact': None}} +2025-01-03 22:33:53,439 DEBUG SenderThread:1983623 [sender.py:send():379] send: files +2025-01-03 22:33:53,440 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json with policy now +2025-01-03 22:33:53,440 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:33:53,440 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:33:53,441 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:33:53,489 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:53,668 INFO wandb-upload_2:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/nyqb6hqz-media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json +2025-01-03 22:33:53,920 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:33:53,920 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:33:54,489 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:55,489 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:56,490 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:57,490 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:58,259 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:33:58,490 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:33:59,490 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:00,490 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:01,101 DEBUG SenderThread:1983623 [sender.py:send():379] send: exit +2025-01-03 22:34:01,101 INFO SenderThread:1983623 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 22:34:01,102 INFO SenderThread:1983623 [sender.py:send_exit():588] handling runtime: 342 +2025-01-03 22:34:01,102 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:34:01,102 INFO SenderThread:1983623 [sender.py:send_exit():594] send defer +2025-01-03 22:34:01,102 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:01,102 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 22:34:01,102 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:01,102 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 22:34:01,102 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 1 +2025-01-03 22:34:01,103 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:01,103 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 22:34:01,103 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:01,103 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 22:34:01,103 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 2 +2025-01-03 22:34:01,103 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:01,103 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 22:34:01,103 INFO HandlerThread:1983623 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 22:34:01,103 DEBUG SystemMonitor:1983623 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 22:34:01,103 DEBUG SystemMonitor:1983623 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 22:34:01,104 INFO HandlerThread:1983623 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 22:34:01,104 INFO HandlerThread:1983623 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 22:34:01,155 INFO HandlerThread:1983623 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 22:34:01,156 INFO HandlerThread:1983623 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 22:34:01,156 INFO HandlerThread:1983623 [interfaces.py:finish():200] Joined network monitor +2025-01-03 22:34:01,156 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:01,156 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 22:34:01,156 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 3 +2025-01-03 22:34:01,156 DEBUG SenderThread:1983623 [sender.py:send():379] send: stats +2025-01-03 22:34:01,156 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:01,156 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 22:34:01,157 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:01,157 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 22:34:01,157 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 4 +2025-01-03 22:34:01,157 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:01,157 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 22:34:01,923 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:34:01,923 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:34:07,698 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:34:07,698 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:34:07,698 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:34:07,699 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:07,699 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 22:34:07,699 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 5 +2025-01-03 22:34:07,699 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:34:07,699 DEBUG SenderThread:1983623 [sender.py:send():379] send: history +2025-01-03 22:34:07,700 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:34:07,700 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:07,700 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 22:34:07,700 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:34:07,701 DEBUG SenderThread:1983623 [sender.py:send():379] send: summary +2025-01-03 22:34:07,704 INFO SenderThread:1983623 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:34:07,705 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:07,705 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 22:34:07,705 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 6 +2025-01-03 22:34:07,705 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:07,705 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 22:34:07,705 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:07,705 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 22:34:07,708 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:34:07,854 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 7 +2025-01-03 22:34:07,854 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:07,854 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 22:34:07,854 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:07,854 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 22:34:07,925 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/config.yaml +2025-01-03 22:34:07,926 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:34:08,102 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:34:09,780 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 8 +2025-01-03 22:34:09,780 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:34:09,780 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:09,780 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 22:34:09,780 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:09,781 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 22:34:09,781 INFO SenderThread:1983623 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 22:34:09,783 INFO SenderThread:1983623 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 22:34:09,785 INFO SenderThread:1983623 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 22:34:09,800 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 9 +2025-01-03 22:34:09,800 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:09,800 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 22:34:09,800 DEBUG SenderThread:1983623 [sender.py:send():379] send: artifact +2025-01-03 22:34:09,926 INFO Thread-12 :1983623 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:34:10,103 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:34:10,147 INFO SenderThread:1983623 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjYyMA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjYyMA==', 'versionIndex': 412}}} +2025-01-03 22:34:10,148 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:10,148 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 22:34:10,148 INFO SenderThread:1983623 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 22:34:10,927 INFO SenderThread:1983623 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files +2025-01-03 22:34:10,927 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/config.yaml config.yaml +2025-01-03 22:34:10,927 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json wandb-summary.json +2025-01-03 22:34:10,928 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:34:10,928 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/requirements.txt requirements.txt +2025-01-03 22:34:10,930 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-metadata.json wandb-metadata.json +2025-01-03 22:34:10,930 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log output.log +2025-01-03 22:34:10,930 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/diff.patch diff.patch +2025-01-03 22:34:10,932 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 22:34:10,933 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/0_eval_0_3b79de18ba2122e2eeca.mp4 media/videos/eval/0_eval_0_3b79de18ba2122e2eeca.mp4 +2025-01-03 22:34:10,933 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/3_eval_3_c408001aca8abf0c7ef9.mp4 media/videos/eval/3_eval_3_c408001aca8abf0c7ef9.mp4 +2025-01-03 22:34:10,934 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/1_eval_1_f5d85a1429df5e5a3e60.mp4 media/videos/eval/1_eval_1_f5d85a1429df5e5a3e60.mp4 +2025-01-03 22:34:10,934 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/2_eval_2_3966ddb8583e8b4d202c.mp4 media/videos/eval/2_eval_2_3966ddb8583e8b4d202c.mp4 +2025-01-03 22:34:10,935 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/videos/eval/4_eval_4_d93a45e1282e6c91c0f9.mp4 media/videos/eval/4_eval_4_d93a45e1282e6c91c0f9.mp4 +2025-01-03 22:34:10,936 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json media/table/eval/success_rate_ci_5_152e8da8630452f2ed70.table.json +2025-01-03 22:34:10,936 INFO SenderThread:1983623 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json media/table/eval/total_success_rate_ci_6_5b1710e6ff2307150c3c.table.json +2025-01-03 22:34:10,936 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 10 +2025-01-03 22:34:10,936 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:34:10,937 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:10,937 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 22:34:10,937 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:10,937 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 22:34:10,937 INFO SenderThread:1983623 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:34:11,103 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:34:11,104 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:34:11,109 INFO wandb-upload_1:1983623 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/config.yaml +2025-01-03 22:34:11,117 INFO wandb-upload_0:1983623 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/wandb-summary.json +2025-01-03 22:34:11,157 INFO wandb-upload_3:1983623 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/requirements.txt +2025-01-03 22:34:11,162 INFO wandb-upload_4:1983623 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/files/output.log +2025-01-03 22:34:11,183 INFO wandb-upload_2:1983623 [upload_job.py:push():130] Uploaded file /tmp/tmpsvnzpgfdwandb/x5izr4j8-events.out.tfevents.1735961299.leonmkim-ROG-Strix-G15CS-G15CS.1983448.0 +2025-01-03 22:34:11,383 INFO Thread-11 (_thread_body):1983623 [sender.py:transition_state():614] send defer: 11 +2025-01-03 22:34:11,383 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:11,383 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 22:34:11,384 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:11,384 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 22:34:11,384 INFO SenderThread:1983623 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 22:34:11,385 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 12 +2025-01-03 22:34:11,385 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:11,385 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 22:34:11,385 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:11,385 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 22:34:11,385 INFO SenderThread:1983623 [file_stream.py:finish():601] file stream finish called +2025-01-03 22:34:11,559 INFO SenderThread:1983623 [file_stream.py:finish():605] file stream finish is done +2025-01-03 22:34:11,559 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 13 +2025-01-03 22:34:11,559 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:11,559 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 22:34:11,559 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:11,559 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 22:34:11,559 INFO SenderThread:1983623 [sender.py:transition_state():614] send defer: 14 +2025-01-03 22:34:11,560 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:34:11,560 INFO HandlerThread:1983623 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 22:34:11,560 DEBUG SenderThread:1983623 [sender.py:send():379] send: final +2025-01-03 22:34:11,560 DEBUG SenderThread:1983623 [sender.py:send():379] send: footer +2025-01-03 22:34:11,560 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: defer +2025-01-03 22:34:11,560 INFO SenderThread:1983623 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 22:34:11,561 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:34:11,561 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 22:34:11,561 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:34:11,561 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:34:11,561 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: server_info +2025-01-03 22:34:11,562 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 22:34:11,564 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 22:34:11,564 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:11,619 DEBUG SenderThread:1983623 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:34:11,619 INFO MainThread:1983623 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 22:34:11,620 INFO MainThread:1983623 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 22:34:11,620 INFO MainThread:1983623 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 22:34:11,620 DEBUG HandlerThread:1983623 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 22:34:11,620 INFO HandlerThread:1983623 [handler.py:finish():882] shutting down handler +2025-01-03 22:34:12,561 INFO WriterThread:1983623 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/run-nhx7r1z0.wandb +2025-01-03 22:34:12,619 INFO SenderThread:1983623 [sender.py:finish():1615] shutting down sender +2025-01-03 22:34:12,620 INFO SenderThread:1983623 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:34:12,620 INFO SenderThread:1983623 [file_pusher.py:join():175] waiting for file pusher diff --git a/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/logs/debug.log b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..153a0f6023548f0b0ea0986c60a9d27eb643f22a --- /dev/null +++ b/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_setup.py:_flush():76] Configure stats pid to 1983448 +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/settings +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/logs/debug.log +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/wandb/run-20250103_222818-nhx7r1z0/logs/debug-internal.log +2025-01-03 22:28:18,550 INFO MainThread:1983448 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:28:18,551 INFO MainThread:1983448 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0'} +2025-01-03 22:28:18,551 INFO MainThread:1983448 [wandb_init.py:init():619] starting backend +2025-01-03 22:28:18,551 INFO MainThread:1983448 [wandb_init.py:init():623] setting up manager +2025-01-03 22:28:18,556 INFO MainThread:1983448 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:28:18,557 INFO MainThread:1983448 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:28:18,570 INFO MainThread:1983448 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:28:18,581 INFO MainThread:1983448 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:28:18,829 INFO MainThread:1983448 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:28:18,901 INFO MainThread:1983448 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:28:18,901 INFO MainThread:1983448 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:28:19,230 INFO MainThread:1983448 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:28:19,230 INFO MainThread:1983448 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:28:19,230 INFO MainThread:1983448 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:28:19,230 INFO MainThread:1983448 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:28:19,232 INFO MainThread:1983448 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:28:19,232 INFO MainThread:1983448 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nhx7r1z0/tb, True +2025-01-03 22:28:24,995 INFO MainThread:1983448 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-03 22:34:12,621 WARNING MsgRouterThr:1983448 [router.py:message_loop():77] message_loop has been closed diff --git a/nyq5rlxn/.hydra/config.yaml b/nyq5rlxn/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50c4489cc892923b6fde889d218eeedfe242c7ee --- /dev/null +++ b/nyq5rlxn/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1922_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/nyq5rlxn/.hydra/hydra.yaml b/nyq5rlxn/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de3692e55c376975e63ca6cb8ead7a1a22b86e99 --- /dev/null +++ b/nyq5rlxn/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/nyq5rlxn + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/nyq5rlxn/.hydra/overrides.yaml b/nyq5rlxn/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/nyq5rlxn/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/nyq5rlxn/eval_policy.log b/nyq5rlxn/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..2f8241392bee764babe19e7e97d3586aaf02e3fa --- /dev/null +++ b/nyq5rlxn/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-04 20:58:13,034][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 20:58:13,037][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-04 20:58:16,218][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/ou8mw55r/eval_robot.log b/ou8mw55r/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..86d7d8a4b9bafede7aebcb3abc25e3e9d9686e64 --- /dev/null +++ b/ou8mw55r/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 19:51:10,764][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 19:51:10,767][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/qulr7kiu/.hydra/config.yaml b/qulr7kiu/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50c4489cc892923b6fde889d218eeedfe242c7ee --- /dev/null +++ b/qulr7kiu/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1922_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/qulr7kiu/.hydra/hydra.yaml b/qulr7kiu/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6cafb411948e1e1538da6817d767e897c1d3543 --- /dev/null +++ b/qulr7kiu/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/qulr7kiu/.hydra/overrides.yaml b/qulr7kiu/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/qulr7kiu/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/qulr7kiu/episode_rosbags/aligned_depth_to_color_K.npy b/qulr7kiu/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/qulr7kiu/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/qulr7kiu/episode_rosbags/cam_tf_world.npy b/qulr7kiu/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/qulr7kiu/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/qulr7kiu/episode_rosbags/color_K.npy b/qulr7kiu/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/qulr7kiu/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/qulr7kiu/episode_rosbags/depth_K.npy b/qulr7kiu/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/qulr7kiu/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/qulr7kiu/eval_robot.log b/qulr7kiu/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..63716993ee23ff9dd5523364114f3efb402f1bcc --- /dev/null +++ b/qulr7kiu/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 22:34:22,562][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 22:34:22,565][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/qulr7kiu/eval_video/0_eval.mp4 b/qulr7kiu/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0c279941f49d39dc9b99dc130d7ec6ab8ba7cac1 --- /dev/null +++ b/qulr7kiu/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdd6e363a52ea071e0fdbd66633825c74224cd431444cb99cd1df4b7f4cb9382 +size 733172 diff --git a/qulr7kiu/eval_video/1_eval.mp4 b/qulr7kiu/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..10634607671054c97c0969a733ae6034cfcb3d2d --- /dev/null +++ b/qulr7kiu/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c428916b5dd1a4c5cc243e4e8fbcff643e248befe72c1208fec7e7d566c7b3a +size 1169557 diff --git a/qulr7kiu/eval_video/2_eval.mp4 b/qulr7kiu/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..990a70447f93421ca547350fdecfa2392bf72367 --- /dev/null +++ b/qulr7kiu/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a6d5b71a8cbdc8ee1f71bb0aacbebe162a8f77a2ae3ed3ef9cf25c14635e28d +size 1195260 diff --git a/qulr7kiu/eval_video/3_eval.mp4 b/qulr7kiu/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e9bbf586509756988134a52468abe9f4524a0d83 --- /dev/null +++ b/qulr7kiu/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5052d6603d25fb3c55cc3cce560e628a3d0a21775a672e5ebee4b704b5efab39 +size 1188321 diff --git a/qulr7kiu/eval_video/4_eval.mp4 b/qulr7kiu/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dc40217e366ce92d35d4afd5eb664067d372a9fb --- /dev/null +++ b/qulr7kiu/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db949b22d5fdd234cf63c7f23cb14cd34ce9b2ce201fd2be2271329eafcc658d +size 1229458 diff --git a/qulr7kiu/tb/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 b/qulr7kiu/tb/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 new file mode 100644 index 0000000000000000000000000000000000000000..9ac3caccd00327a15802dd8a4c7d839b2c1f0dd9 --- /dev/null +++ b/qulr7kiu/tb/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c84e28af72660f677b7718c8041a32695cf14ae699c37361922b0b41d620137 +size 6519 diff --git a/qulr7kiu/wandb/debug-internal.log b/qulr7kiu/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ccb63a5e9c2b7010159e48a1549f925c66b8b731 --- /dev/null +++ b/qulr7kiu/wandb/debug-internal.log @@ -0,0 +1,757 @@ +2025-01-03 22:34:29,346 INFO StreamThr :1984755 [internal.py:wandb_internal():85] W&B internal server running at pid: 1984755, started at: 2025-01-03 22:34:29.345769 +2025-01-03 22:34:29,347 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:34:29,357 INFO WriterThread:1984755 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/run-qulr7kiu.wandb +2025-01-03 22:34:29,358 DEBUG SenderThread:1984755 [sender.py:send():379] send: header +2025-01-03 22:34:29,365 DEBUG SenderThread:1984755 [sender.py:send():379] send: run +2025-01-03 22:34:29,612 INFO SenderThread:1984755 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files +2025-01-03 22:34:29,612 INFO SenderThread:1984755 [sender.py:_start_run_threads():1188] run started: qulr7kiu with start time 1735961669.346192 +2025-01-03 22:34:29,629 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:34:29,629 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:34:29,700 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:34:29,705 DEBUG HandlerThread:1984755 [system_info.py:__init__():26] System info init +2025-01-03 22:34:29,705 DEBUG HandlerThread:1984755 [system_info.py:__init__():41] System info init done +2025-01-03 22:34:29,705 INFO HandlerThread:1984755 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:34:29,705 INFO SystemMonitor:1984755 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:34:29,706 INFO HandlerThread:1984755 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:34:29,706 INFO SystemMonitor:1984755 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:34:29,707 INFO SystemMonitor:1984755 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:34:29,708 INFO SystemMonitor:1984755 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:34:29,708 INFO SystemMonitor:1984755 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:34:29,709 INFO SystemMonitor:1984755 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:34:29,719 DEBUG HandlerThread:1984755 [system_info.py:probe():152] Probing system +2025-01-03 22:34:29,720 DEBUG HandlerThread:1984755 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:34:29,725 DEBUG HandlerThread:1984755 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:34:29,725 DEBUG HandlerThread:1984755 [system_info.py:probe():200] Probing system done +2025-01-03 22:34:29,725 DEBUG HandlerThread:1984755 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:34:29.719513', 'startedAt': '2025-01-04T03:34:29.334534', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.3318750000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4497.391, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.952, 'min': 800.0, 'max': 4700.0}, {'current': 4500.014, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 851.2897453308105}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:34:29,726 INFO HandlerThread:1984755 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:34:29,726 INFO HandlerThread:1984755 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:34:29,726 DEBUG HandlerThread:1984755 [system_info.py:_save_code():44] Saving code +2025-01-03 22:34:29,727 DEBUG HandlerThread:1984755 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:34:29,727 DEBUG HandlerThread:1984755 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:34:29,790 DEBUG HandlerThread:1984755 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:34:29,791 INFO HandlerThread:1984755 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:34:29,794 DEBUG SenderThread:1984755 [sender.py:send():379] send: files +2025-01-03 22:34:29,794 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:34:29,794 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:34:29,794 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:34:30,024 INFO wandb-upload_0:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/dypoc7d8-wandb-metadata.json +2025-01-03 22:34:30,029 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:34:30,029 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:34:30,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:30,031 DEBUG SenderThread:1984755 [sender.py:send():379] send: telemetry +2025-01-03 22:34:30,065 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/rfojwaap-diff.patch +2025-01-03 22:34:30,073 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:34:30,073 INFO HandlerThread:1984755 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/tb" + save: true +} +control { + relay_id: "265f69dd70ed47ba83a6b9d5dd2fa93c" +} +_info { + stream_id: "qulr7kiu" +} + +2025-01-03 22:34:30,073 DEBUG HandlerThread:1984755 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:34:30,074 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:34:30,232 DEBUG SenderThread:1984755 [sender.py:send():379] send: tbrecord +2025-01-03 22:34:30,233 DEBUG SenderThread:1984755 [sender.py:send():379] send: files +2025-01-03 22:34:30,233 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 with policy live +2025-01-03 22:34:30,451 INFO wandb-upload_0:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/7k8cpf6i-events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:34:30,613 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/code/FISH/eval_robot.py +2025-01-03 22:34:30,613 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-metadata.json +2025-01-03 22:34:30,614 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:34:30,614 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/requirements.txt +2025-01-03 22:34:30,614 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:34:30,614 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/diff.patch +2025-01-03 22:34:30,614 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/code +2025-01-03 22:34:30,614 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/code/FISH +2025-01-03 22:34:31,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:32,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:32,613 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:34:33,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:34,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:34,550 INFO wandb-upload_1:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/2r3f18u4-code/FISH/eval_robot.py +2025-01-03 22:34:35,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:34:35,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:36,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:36,926 DEBUG SenderThread:1984755 [sender.py:send():379] send: config +2025-01-03 22:34:37,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:38,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:38,615 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:34:39,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:40,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:34:40,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:41,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:42,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:42,616 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:34:43,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:44,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:45,029 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:34:45,030 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:34:45,032 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:45,189 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:34:46,033 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:46,617 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:34:47,032 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:48,033 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:48,618 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:34:49,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:50,033 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:50,618 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:34:51,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:34:51,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:52,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:52,619 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:34:53,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:54,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:55,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:56,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:34:56,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:57,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:58,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:59,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:00,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:35:00,030 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:35:00,073 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:01,035 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:02,035 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:03,035 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:04,035 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:04,689 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:05,035 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:05,622 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/config.yaml +2025-01-03 22:35:06,035 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:07,035 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:07,932 DEBUG SenderThread:1984755 [sender.py:send():379] send: files +2025-01-03 22:35:07,932 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_fdd6e363a52ea071e0fd.mp4 with policy now +2025-01-03 22:35:07,932 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:35:07,933 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:35:07,934 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:35:07,936 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:35:08,036 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:08,138 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/tk5x1qa7-media/videos/eval/0_eval_0_fdd6e363a52ea071e0fd.mp4 +2025-01-03 22:35:08,623 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:35:08,623 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:35:08,623 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/0_eval_0_fdd6e363a52ea071e0fd.mp4 +2025-01-03 22:35:08,623 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval +2025-01-03 22:35:08,623 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos +2025-01-03 22:35:08,624 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media +2025-01-03 22:35:08,870 INFO wandb-upload_1:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/62fkgfei-media/videos/eval/0_eval_0_fdd6e363a52ea071e0fd.mp4 +2025-01-03 22:35:09,039 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:09,953 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:10,037 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:10,623 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:35:11,037 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:12,037 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:12,624 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:35:13,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:14,038 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:15,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:35:15,030 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:35:15,073 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:15,093 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:16,038 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:16,625 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:35:17,039 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:18,039 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:19,039 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:20,039 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:21,032 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:21,039 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:22,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:23,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:24,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:25,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:26,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:27,032 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:27,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:28,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:29,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:29,709 DEBUG SystemMonitor:1984755 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:35:29,710 DEBUG SenderThread:1984755 [sender.py:send():379] send: stats +2025-01-03 22:35:30,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:35:30,030 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:35:30,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:30,630 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:35:30,630 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:35:30,991 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/910defue-events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:35:31,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:32,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:32,631 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:35:32,725 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:33,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:34,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:34,631 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:35:35,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:36,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:37,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:38,032 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:38,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:39,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:40,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:41,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:42,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:43,032 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:43,042 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:44,042 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:45,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:35:45,030 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:35:45,073 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:46,042 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:47,042 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:48,042 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:48,158 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:49,042 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:50,042 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:51,042 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:52,042 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:53,043 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:54,033 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:54,043 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:55,043 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:56,043 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:57,043 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:58,043 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:58,638 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:35:59,043 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:59,213 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:59,711 DEBUG SenderThread:1984755 [sender.py:send():379] send: stats +2025-01-03 22:36:00,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:36:00,030 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:36:00,077 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:01,044 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:01,678 DEBUG SenderThread:1984755 [sender.py:send():379] send: files +2025-01-03 22:36:01,678 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_2c428916b5dd1a4c5cc2.mp4 with policy now +2025-01-03 22:36:01,686 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:36:01,686 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:36:01,686 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:36:01,693 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:36:01,882 INFO wandb-upload_1:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/i692njwn-media/videos/eval/1_eval_1_2c428916b5dd1a4c5cc2.mp4 +2025-01-03 22:36:02,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:02,640 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:02,640 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:36:02,640 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/1_eval_1_2c428916b5dd1a4c5cc2.mp4 +2025-01-03 22:36:02,640 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval +2025-01-03 22:36:02,903 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/24jemtfr-media/videos/eval/1_eval_1_2c428916b5dd1a4c5cc2.mp4 +2025-01-03 22:36:03,044 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:04,045 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:04,640 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:04,721 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:05,049 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:06,047 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:06,641 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:07,049 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:08,053 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:09,045 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:10,037 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:10,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:10,642 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:11,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:12,045 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:13,045 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:13,643 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:36:14,045 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:14,465 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:36:14,465 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:36:14,466 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:36:14,644 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:14,644 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:36:15,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:36:15,030 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:36:15,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:15,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:16,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:16,644 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:17,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:18,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:18,645 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:19,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:20,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:21,038 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:21,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:22,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:23,049 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:24,049 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:25,049 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:26,049 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:27,037 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:27,049 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:28,051 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:29,052 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:29,711 DEBUG SenderThread:1984755 [sender.py:send():379] send: stats +2025-01-03 22:36:30,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:36:30,030 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:36:30,073 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:31,052 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:32,052 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:32,110 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:33,052 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:34,052 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:35,052 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:36,052 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:36,651 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:37,053 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:37,530 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:38,053 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:39,054 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:40,054 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:41,061 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:42,061 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:43,038 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:43,061 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:44,069 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:45,036 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:36:45,036 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:36:45,070 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:45,346 DEBUG SenderThread:1984755 [sender.py:send():379] send: files +2025-01-03 22:36:45,346 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_0a6d5b71a8cbdc8ee1f7.mp4 with policy now +2025-01-03 22:36:45,355 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:36:45,355 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:36:45,355 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:36:45,357 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:36:45,556 INFO wandb-upload_1:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/z15j1f3k-media/videos/eval/2_eval_2_0a6d5b71a8cbdc8ee1f7.mp4 +2025-01-03 22:36:45,658 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:36:45,658 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/2_eval_2_0a6d5b71a8cbdc8ee1f7.mp4 +2025-01-03 22:36:45,658 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval +2025-01-03 22:36:45,939 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/mh60bmdo-media/videos/eval/2_eval_2_0a6d5b71a8cbdc8ee1f7.mp4 +2025-01-03 22:36:46,070 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:46,658 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:47,071 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:48,069 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:48,385 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:49,072 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:50,072 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:50,660 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:51,072 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:52,071 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:52,660 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:53,074 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:53,829 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:54,072 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:55,074 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:56,072 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:57,074 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:58,075 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:59,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:59,077 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:59,711 DEBUG SenderThread:1984755 [sender.py:send():379] send: stats +2025-01-03 22:37:00,036 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:37:00,036 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:37:00,077 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:01,080 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:02,078 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:03,076 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:04,080 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:04,098 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:05,078 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:06,109 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:06,311 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:37:06,311 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:37:06,311 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:37:06,665 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:37:06,665 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:37:06,907 INFO wandb-upload_1:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/sz3ym9u1-events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:37:07,087 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:08,084 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:08,665 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:37:09,084 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:09,137 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:10,089 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:10,666 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:37:11,083 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:12,083 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:12,667 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:37:13,084 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:14,083 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:14,154 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:15,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:37:15,034 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:37:15,083 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:16,084 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:17,084 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:18,084 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:19,084 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:19,159 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:20,084 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:21,085 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:22,085 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:23,085 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:24,085 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:25,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:25,085 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:26,085 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:27,086 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:28,086 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:29,086 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:29,712 DEBUG SenderThread:1984755 [sender.py:send():379] send: stats +2025-01-03 22:37:30,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:37:30,034 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:37:30,086 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:30,157 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:31,086 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:32,086 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:33,086 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:34,086 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:35,086 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:36,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:36,087 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:37,088 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:38,092 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:38,145 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:37:38,145 DEBUG SenderThread:1984755 [sender.py:send():379] send: files +2025-01-03 22:37:38,146 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_5052d6603d25fb3c55cc.mp4 with policy now +2025-01-03 22:37:38,146 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:37:38,146 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:37:38,147 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:37:38,394 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/6g0cs19f-media/videos/eval/3_eval_3_5052d6603d25fb3c55cc.mp4 +2025-01-03 22:37:38,676 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:37:38,676 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:37:38,676 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/3_eval_3_5052d6603d25fb3c55cc.mp4 +2025-01-03 22:37:38,676 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval +2025-01-03 22:37:38,947 INFO wandb-upload_1:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/oind4h1b-media/videos/eval/3_eval_3_5052d6603d25fb3c55cc.mp4 +2025-01-03 22:37:39,096 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:40,093 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:40,676 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:37:41,047 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:41,095 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:42,098 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:42,677 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:37:43,101 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:44,107 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:45,050 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:37:45,050 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:37:45,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:46,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:46,146 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:46,678 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:37:46,679 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:37:47,115 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:47,359 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:37:47,360 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:37:47,360 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:37:47,679 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:37:47,679 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:37:48,115 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:48,679 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:37:49,110 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:50,113 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:51,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:51,955 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:52,110 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:52,680 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:37:53,110 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:54,110 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:55,110 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:56,110 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:57,045 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:57,110 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:58,110 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:59,111 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:59,713 DEBUG SenderThread:1984755 [sender.py:send():379] send: stats +2025-01-03 22:38:00,044 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:38:00,045 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:38:00,111 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:01,111 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:02,111 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:02,137 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:03,111 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:04,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:05,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:06,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:07,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:08,048 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:08,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:09,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:10,113 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:11,113 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:12,113 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:12,688 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:38:13,113 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:13,949 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:14,113 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:15,044 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:38:15,045 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:38:15,114 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:16,114 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:17,114 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:18,139 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:18,690 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:38:18,834 DEBUG SenderThread:1984755 [sender.py:send():379] send: files +2025-01-03 22:38:18,834 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_db949b22d5fdd234cf63.mp4 with policy now +2025-01-03 22:38:18,840 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:38:18,840 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:38:18,840 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:18,841 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:19,039 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/b1l0r5zs-media/videos/eval/4_eval_4_db949b22d5fdd234cf63.mp4 +2025-01-03 22:38:19,129 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:19,690 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:38:19,690 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/4_eval_4_db949b22d5fdd234cf63.mp4 +2025-01-03 22:38:19,690 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval +2025-01-03 22:38:19,865 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:19,957 INFO wandb-upload_1:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/yhytphbf-media/videos/eval/4_eval_4_db949b22d5fdd234cf63.mp4 +2025-01-03 22:38:20,131 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:20,690 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:38:21,131 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:22,131 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:23,130 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:24,127 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:24,692 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:38:25,051 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:25,128 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:26,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:26,692 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:38:27,145 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:28,138 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:28,406 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:38:28,407 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:28,407 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:28,693 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:38:28,693 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:38:28,833 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:38:28,833 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:38:28,924 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/bkk6vjv9-events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:38:29,101 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:38:29,132 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:29,326 INFO wandb-upload_1:1984755 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp7qpap0mi +2025-01-03 22:38:29,486 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:38:29,494 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:38:29,694 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:38:29,694 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/table/eval/total_success_rate_ci_6_5c9c8917af98e9d22a01.table.json +2025-01-03 22:38:29,694 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json +2025-01-03 22:38:29,694 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media +2025-01-03 22:38:29,694 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/table +2025-01-03 22:38:29,694 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/table/eval +2025-01-03 22:38:29,762 INFO SenderThread:1984755 [sender.py:send_request_log_artifact():1525] logged artifact run-qulr7kiu-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk4NDM2MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4OTIxMg==', 'latestArtifact': None}} +2025-01-03 22:38:29,762 DEBUG SenderThread:1984755 [sender.py:send():379] send: files +2025-01-03 22:38:29,762 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json with policy now +2025-01-03 22:38:29,762 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:38:29,763 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,763 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,763 DEBUG SenderThread:1984755 [sender.py:send():379] send: telemetry +2025-01-03 22:38:29,763 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,764 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,764 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,764 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,764 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,764 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,765 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,765 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,765 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,766 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,766 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,766 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,766 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,766 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,767 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,767 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,767 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,767 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,767 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,768 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,768 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,768 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,768 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:38:29,936 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/kgw2mw0a-media/table/eval/total_success_rate_ci_6_5c9c8917af98e9d22a01.table.json +2025-01-03 22:38:29,955 INFO wandb-upload_0:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/1y2cr3kt-media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json +2025-01-03 22:38:30,045 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:38:30,132 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:30,142 INFO wandb-upload_1:1984755 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpq3o80krz +2025-01-03 22:38:30,189 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/uh4fh8i0-media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json +2025-01-03 22:38:30,694 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:38:30,694 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:38:30,703 INFO SenderThread:1984755 [sender.py:send_request_log_artifact():1525] logged artifact run-qulr7kiu-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk4NDM2OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4OTIxNA==', 'latestArtifact': None}} +2025-01-03 22:38:30,703 DEBUG SenderThread:1984755 [sender.py:send():379] send: files +2025-01-03 22:38:30,703 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_5c9c8917af98e9d22a01.table.json with policy now +2025-01-03 22:38:30,703 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:38:30,703 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:30,704 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:30,704 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:30,704 DEBUG SenderThread:1984755 [sender.py:send():379] send: stats +2025-01-03 22:38:30,705 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:38:30,938 INFO wandb-upload_0:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/9mnrah13-media/table/eval/total_success_rate_ci_6_5c9c8917af98e9d22a01.table.json +2025-01-03 22:38:31,132 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:31,695 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:38:32,132 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:33,132 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:34,132 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:35,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:36,050 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:36,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:37,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:38,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:39,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:40,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:41,050 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:41,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:42,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:43,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:44,134 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:45,045 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:38:45,045 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:38:45,134 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:46,134 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:46,636 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:47,134 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:48,134 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:49,135 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:49,270 DEBUG SenderThread:1984755 [sender.py:send():379] send: exit +2025-01-03 22:38:49,270 INFO SenderThread:1984755 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 22:38:49,270 INFO SenderThread:1984755 [sender.py:send_exit():588] handling runtime: 259 +2025-01-03 22:38:49,271 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:49,271 INFO SenderThread:1984755 [sender.py:send_exit():594] send defer +2025-01-03 22:38:49,271 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:49,271 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 22:38:49,272 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:49,272 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 22:38:49,272 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 1 +2025-01-03 22:38:49,272 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:49,272 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 22:38:49,272 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:49,272 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 22:38:49,272 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 2 +2025-01-03 22:38:49,272 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:49,272 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 22:38:49,272 INFO HandlerThread:1984755 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 22:38:49,273 DEBUG SystemMonitor:1984755 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 22:38:49,273 DEBUG SystemMonitor:1984755 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 22:38:49,273 INFO HandlerThread:1984755 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 22:38:49,273 INFO HandlerThread:1984755 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 22:38:49,324 INFO HandlerThread:1984755 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 22:38:49,324 INFO HandlerThread:1984755 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 22:38:49,324 INFO HandlerThread:1984755 [interfaces.py:finish():200] Joined network monitor +2025-01-03 22:38:49,324 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:49,324 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 22:38:49,325 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 3 +2025-01-03 22:38:49,325 DEBUG SenderThread:1984755 [sender.py:send():379] send: stats +2025-01-03 22:38:49,325 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:49,325 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 22:38:49,325 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:49,325 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 22:38:49,325 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 4 +2025-01-03 22:38:49,325 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:49,325 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 22:38:49,702 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:38:50,702 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:38:52,703 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/config.yaml +2025-01-03 22:38:55,422 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:38:55,423 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:55,423 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:38:55,424 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:55,424 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 22:38:55,424 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 5 +2025-01-03 22:38:55,424 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:38:55,424 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:38:55,424 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:55,425 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:55,426 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 22:38:55,426 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:55,426 DEBUG SenderThread:1984755 [sender.py:send():379] send: summary +2025-01-03 22:38:55,430 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:55,430 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:55,430 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 22:38:55,430 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 6 +2025-01-03 22:38:55,430 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:55,430 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 22:38:55,430 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:55,430 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 22:38:55,431 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 7 +2025-01-03 22:38:55,431 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:55,431 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:55,431 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 22:38:55,431 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:55,431 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 22:38:55,704 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:38:56,271 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:38:57,869 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 8 +2025-01-03 22:38:57,869 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:38:57,869 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:57,869 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 22:38:57,870 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:57,870 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 22:38:57,870 INFO SenderThread:1984755 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 22:38:57,874 INFO SenderThread:1984755 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 22:38:57,876 INFO SenderThread:1984755 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 22:38:57,890 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 9 +2025-01-03 22:38:57,890 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:57,890 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 22:38:57,890 DEBUG SenderThread:1984755 [sender.py:send():379] send: artifact +2025-01-03 22:38:58,118 INFO SenderThread:1984755 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjYyMA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjYyMA==', 'versionIndex': 412}}} +2025-01-03 22:38:58,118 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:58,118 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 22:38:58,118 INFO SenderThread:1984755 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 22:38:58,272 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:38:58,706 INFO SenderThread:1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:38:58,706 INFO SenderThread:1984755 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files +2025-01-03 22:38:58,706 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/config.yaml config.yaml +2025-01-03 22:38:58,706 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json wandb-summary.json +2025-01-03 22:38:58,708 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/requirements.txt requirements.txt +2025-01-03 22:38:58,708 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-metadata.json wandb-metadata.json +2025-01-03 22:38:58,708 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:38:58,711 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log output.log +2025-01-03 22:38:58,712 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/diff.patch diff.patch +2025-01-03 22:38:58,712 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 22:38:58,712 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/2_eval_2_0a6d5b71a8cbdc8ee1f7.mp4 media/videos/eval/2_eval_2_0a6d5b71a8cbdc8ee1f7.mp4 +2025-01-03 22:38:58,712 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/4_eval_4_db949b22d5fdd234cf63.mp4 media/videos/eval/4_eval_4_db949b22d5fdd234cf63.mp4 +2025-01-03 22:38:58,712 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/0_eval_0_fdd6e363a52ea071e0fd.mp4 media/videos/eval/0_eval_0_fdd6e363a52ea071e0fd.mp4 +2025-01-03 22:38:58,712 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/3_eval_3_5052d6603d25fb3c55cc.mp4 media/videos/eval/3_eval_3_5052d6603d25fb3c55cc.mp4 +2025-01-03 22:38:58,712 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/1_eval_1_2c428916b5dd1a4c5cc2.mp4 media/videos/eval/1_eval_1_2c428916b5dd1a4c5cc2.mp4 +2025-01-03 22:38:58,712 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json +2025-01-03 22:38:58,713 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/table/eval/total_success_rate_ci_6_5c9c8917af98e9d22a01.table.json media/table/eval/total_success_rate_ci_6_5c9c8917af98e9d22a01.table.json +2025-01-03 22:38:58,713 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 10 +2025-01-03 22:38:58,713 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:38:58,713 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:58,714 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 22:38:58,715 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:58,715 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 22:38:58,716 INFO SenderThread:1984755 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:38:58,878 INFO wandb-upload_0:1984755 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:38:58,889 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/config.yaml +2025-01-03 22:38:58,978 INFO wandb-upload_1:1984755 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/requirements.txt +2025-01-03 22:38:58,996 INFO wandb-upload_3:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/bw9txxql-events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:38:58,998 INFO wandb-upload_4:1984755 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:38:59,198 INFO Thread-11 (_thread_body):1984755 [sender.py:transition_state():614] send defer: 11 +2025-01-03 22:38:59,198 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:59,198 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 22:38:59,199 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:59,199 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 22:38:59,199 INFO SenderThread:1984755 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 22:38:59,200 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 12 +2025-01-03 22:38:59,200 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:59,200 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 22:38:59,200 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:59,201 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 22:38:59,201 INFO SenderThread:1984755 [file_stream.py:finish():601] file stream finish called +2025-01-03 22:38:59,272 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:38:59,390 INFO SenderThread:1984755 [file_stream.py:finish():605] file stream finish is done +2025-01-03 22:38:59,390 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 13 +2025-01-03 22:38:59,390 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:38:59,390 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:59,390 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 22:38:59,390 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:59,391 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 22:38:59,391 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 14 +2025-01-03 22:38:59,391 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:59,391 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 22:38:59,391 DEBUG SenderThread:1984755 [sender.py:send():379] send: final +2025-01-03 22:38:59,391 DEBUG SenderThread:1984755 [sender.py:send():379] send: footer +2025-01-03 22:38:59,391 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:59,391 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 22:38:59,391 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:38:59,392 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:38:59,392 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 22:38:59,392 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:38:59,392 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 22:38:59,393 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 22:38:59,393 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:59,393 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: server_info +2025-01-03 22:38:59,457 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:38:59,458 INFO MainThread:1984755 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 22:38:59,458 INFO MainThread:1984755 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 22:38:59,458 INFO MainThread:1984755 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 22:38:59,458 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 22:38:59,458 INFO HandlerThread:1984755 [handler.py:finish():882] shutting down handler +2025-01-03 22:39:00,393 INFO WriterThread:1984755 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/run-qulr7kiu.wandb +2025-01-03 22:39:00,458 INFO SenderThread:1984755 [sender.py:finish():1615] shutting down sender +2025-01-03 22:39:00,458 INFO SenderThread:1984755 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:39:00,458 INFO SenderThread:1984755 [file_pusher.py:join():175] waiting for file pusher diff --git a/qulr7kiu/wandb/debug.log b/qulr7kiu/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..5340f2ac9cc9f8d36727cb8004a48ff88df697c5 --- /dev/null +++ b/qulr7kiu/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:34:29,335 INFO MainThread:1984590 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:34:29,335 INFO MainThread:1984590 [wandb_setup.py:_flush():76] Configure stats pid to 1984590 +2025-01-03 22:34:29,335 INFO MainThread:1984590 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/settings +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/logs/debug.log +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/logs/debug-internal.log +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu'} +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_init.py:init():619] starting backend +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_init.py:init():623] setting up manager +2025-01-03 22:34:29,344 INFO MainThread:1984590 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:34:29,345 INFO MainThread:1984590 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:34:29,357 INFO MainThread:1984590 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:34:29,363 INFO MainThread:1984590 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:34:29,629 INFO MainThread:1984590 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:34:29,694 INFO MainThread:1984590 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:34:29,695 INFO MainThread:1984590 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:34:30,030 INFO MainThread:1984590 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:34:30,030 INFO MainThread:1984590 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:34:30,030 INFO MainThread:1984590 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:34:30,030 INFO MainThread:1984590 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:34:30,032 INFO MainThread:1984590 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:34:30,032 INFO MainThread:1984590 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/tb, True +2025-01-03 22:34:36,926 INFO MainThread:1984590 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-03 22:39:00,459 WARNING MsgRouterThr:1984590 [router.py:message_loop():77] message_loop has been closed diff --git a/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/code/FISH/eval_robot.py b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/config.yaml b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f2b484e896826a0b2988096f890b2ebee4a0188 --- /dev/null +++ b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1922_1' +wandb_notes: + desc: null + value: 1922_1_restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735961669 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lady +left_book_slot: + desc: null + value: twodim diff --git a/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/diff.patch b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b8501904dd167f150d5cfd5100b04efef6d61ed1 --- /dev/null +++ b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..d2afe29 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json new file mode 100644 index 0000000000000000000000000000000000000000..487ca02c66feb7a1705ed4827a7a884ec91cf1f6 --- /dev/null +++ b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.2, 0.023880511617516982, 0.7333634495268442, 0.015644066132618713, 0.6888955831149919, 0.02994169304235303, 0.5159374773470475, 0.021239768809464744, 0.4524847120141646]]} \ No newline at end of file diff --git a/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/table/eval/total_success_rate_ci_6_5c9c8917af98e9d22a01.table.json b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/table/eval/total_success_rate_ci_6_5c9c8917af98e9d22a01.table.json new file mode 100644 index 0000000000000000000000000000000000000000..e4d719efaed6d79f95bf46aa89118aa1a2c63803 --- /dev/null +++ b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/table/eval/total_success_rate_ci_6_5c9c8917af98e9d22a01.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[23, 30, 0.7666666666666667, 0.579828977568873, 0.9037904440549078, 0.5848899036476107, 0.8843533768481209, 0.6088560074350085, 0.8762526027105753, 0.6711570508435782, 0.858932282703854]]} \ No newline at end of file diff --git a/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/0_eval_0_fdd6e363a52ea071e0fd.mp4 b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/0_eval_0_fdd6e363a52ea071e0fd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0c279941f49d39dc9b99dc130d7ec6ab8ba7cac1 --- /dev/null +++ b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/0_eval_0_fdd6e363a52ea071e0fd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdd6e363a52ea071e0fdbd66633825c74224cd431444cb99cd1df4b7f4cb9382 +size 733172 diff --git a/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/1_eval_1_2c428916b5dd1a4c5cc2.mp4 b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/1_eval_1_2c428916b5dd1a4c5cc2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..10634607671054c97c0969a733ae6034cfcb3d2d --- /dev/null +++ b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/1_eval_1_2c428916b5dd1a4c5cc2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c428916b5dd1a4c5cc243e4e8fbcff643e248befe72c1208fec7e7d566c7b3a +size 1169557 diff --git a/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/2_eval_2_0a6d5b71a8cbdc8ee1f7.mp4 b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/2_eval_2_0a6d5b71a8cbdc8ee1f7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..990a70447f93421ca547350fdecfa2392bf72367 --- /dev/null +++ b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/2_eval_2_0a6d5b71a8cbdc8ee1f7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a6d5b71a8cbdc8ee1f71bb0aacbebe162a8f77a2ae3ed3ef9cf25c14635e28d +size 1195260 diff --git a/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/3_eval_3_5052d6603d25fb3c55cc.mp4 b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/3_eval_3_5052d6603d25fb3c55cc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e9bbf586509756988134a52468abe9f4524a0d83 --- /dev/null +++ b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/3_eval_3_5052d6603d25fb3c55cc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5052d6603d25fb3c55cc3cce560e628a3d0a21775a672e5ebee4b704b5efab39 +size 1188321 diff --git a/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/4_eval_4_db949b22d5fdd234cf63.mp4 b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/4_eval_4_db949b22d5fdd234cf63.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dc40217e366ce92d35d4afd5eb664067d372a9fb --- /dev/null +++ b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/4_eval_4_db949b22d5fdd234cf63.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db949b22d5fdd234cf63c7f23cb14cd34ce9b2ce201fd2be2271329eafcc658d +size 1229458 diff --git a/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..645f1f5a7a0845abb52ca6194aeb5bdf83dfe28a --- /dev/null +++ b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log @@ -0,0 +1,267 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735961676.926940]: resetting environment +[INFO] [1735961676.930382]: cleared current plan +[INFO] [1735961676.931504]: moving to home +[INFO] [1735961677.933257]: reached home +[INFO] [1735961677.934794]: reset action history +[INFO] [1735961680.646118]: environment reset +Starting episode 0 +[INFO] [1735961680.647936]: resetting environment +[INFO] [1735961680.650706]: cleared current plan +[INFO] [1735961680.651570]: moving to home +[INFO] [1735961681.653263]: reached home +[INFO] [1735961681.654661]: reset action history +[INFO] [1735961684.362308]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735961687.342388]: resetting environment +[INFO] [1735961687.346495]: cleared current plan +[INFO] [1735961687.347506]: moving to home +[INFO] [1735961688.349370]: reached home +[INFO] [1735961688.350497]: reset action history +[INFO] [1735961691.057892]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/episode_rosbags/episode_0_2025-01-03-22-34-51.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1984590.log +started roslaunch server http://158.130.50.37:45399/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1984880] +process[print_text-2]: started with pid [1984881] +process[rosbag_record-3]: started with pid [1984882] +started bagging! +preempting episode +[INFO] [1735961706.879095]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735961707.681182]: Stored episode 1. +[INFO] [1735961707.681714]: Saving video... +[INFO] [1735961707.908059]: Video saved! +Starting episode 1 +[INFO] [1735961707.933623]: resetting environment +[INFO] [1735961707.951801]: cleared current plan +[INFO] [1735961707.952658]: moving to home +[INFO] [1735961711.155093]: reached home +[INFO] [1735961711.156093]: reset action history +[INFO] [1735961713.863338]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735961729.681176]: resetting environment +[INFO] [1735961729.683364]: cleared current plan +[INFO] [1735961729.683531]: moving to home +[INFO] [1735961730.684958]: reached home +[INFO] [1735961730.685378]: reset action history +[INFO] [1735961733.392481]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/episode_rosbags/episode_1_2025-01-03-22-35-33.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1984590.log +started roslaunch server http://158.130.50.37:38169/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1984991] +process[print_text-5]: started with pid [1984992] +process[rosbag_record-6]: started with pid [1984993] +started bagging! +For topic gripper_width: timestamp difference is 106571333 for nearest: 1735961756161530630 - target: 1735961756054959297 at idx 249 +[INFO] [1735961760.355038]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735961761.114100]: Stored episode 2. +[INFO] [1735961761.114373]: Saving video... +[INFO] [1735961761.637541]: Video saved! +Starting episode 2 +[INFO] [1735961761.693518]: resetting environment +[INFO] [1735961761.708301]: cleared current plan +[INFO] [1735961761.717348]: moving to home +[WARN] [1735961761.700432]: Plan exhausted +[INFO] [1735961765.233314]: reached home +[INFO] [1735961765.234570]: reset action history +[INFO] [1735961768.011477]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735961773.578628]: resetting environment +[INFO] [1735961773.590335]: cleared current plan +[INFO] [1735961773.592278]: moving to home +[INFO] [1735961774.593308]: reached home +[INFO] [1735961774.594664]: reset action history +[INFO] [1735961777.302114]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/episode_rosbags/episode_2_2025-01-03-22-36-17.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1984590.log +started roslaunch server http://158.130.50.37:44847/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1985130] +process[print_text-8]: started with pid [1985131] +process[rosbag_record-9]: started with pid [1985155] +started bagging! +For topic gripper_width: timestamp difference is 104779486 for nearest: 1735961795328195576 - target: 1735961795223416090 at idx 238 +For topic gripper_width: timestamp difference is 104947582 for nearest: 1735961795528195635 - target: 1735961795423248053 at idx 240 +[INFO] [1735961804.290340]: Storing episode... +[print_text-8] killing on exit +[pub_text-7] killing on exit +[rosbag_record-9] killing on exit +[INFO] [1735961805.037784]: Stored episode 3. +[INFO] [1735961805.042088]: Saving video... +[INFO] [1735961805.299956]: Video saved! +Starting episode 3 +[INFO] [1735961805.366275]: resetting environment +[INFO] [1735961805.383203]: cleared current plan +[INFO] [1735961805.384012]: moving to home +[INFO] [1735961808.995713]: reached home +[INFO] [1735961809.000171]: reset action history +[INFO] [1735961811.783044]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735961826.077795]: resetting environment +[INFO] [1735961826.089240]: cleared current plan +[INFO] [1735961826.089697]: moving to home +[INFO] [1735961827.094659]: reached home +[INFO] [1735961827.101038]: reset action history +[INFO] [1735961829.884113]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/episode_rosbags/episode_3_2025-01-03-22-37-09.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1984590.log +started roslaunch server http://158.130.50.37:43013/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1985268] +process[print_text-11]: started with pid [1985269] +process[rosbag_record-12]: started with pid [1985293] +started bagging! +[INFO] [1735961856.853298]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1735961857.636229]: Stored episode 4. +[INFO] [1735961857.639375]: Saving video... +[INFO] [1735961858.097856]: Video saved! +Starting episode 4 +[INFO] [1735961858.142681]: resetting environment +[INFO] [1735961858.151030]: cleared current plan +[INFO] [1735961858.151375]: moving to home +[INFO] [1735961861.655409]: reached home +[INFO] [1735961861.660807]: reset action history +[INFO] [1735961864.494364]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735961866.650314]: resetting environment +[INFO] [1735961866.664251]: cleared current plan +[INFO] [1735961866.676145]: moving to home +[INFO] [1735961867.683345]: reached home +[INFO] [1735961867.686170]: reset action history +[INFO] [1735961870.455544]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/episode_rosbags/episode_4_2025-01-03-22-37-50.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1984590.log +started roslaunch server http://158.130.50.37:46423/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1985424] +process[print_text-14]: started with pid [1985425] +process[rosbag_record-15]: started with pid [1985426] +started bagging! +For topic gripper_width: timestamp difference is -252223040 for nearest: 1735961891561536049 - target: 1735961891813759089 at idx 249 +[INFO] [1735961897.699007]: Storing episode... +[rosbag_record-15] killing on exit[pub_text-13] killing on exit +[print_text-14] killing on exit +preempting episode +[INFO] [1735961898.444847]: Stored episode 5. +[INFO] [1735961898.456688]: Saving video... +[INFO] [1735961898.776989]: Video saved! +[INFO] [1735961898.841435]: resetting environment +[INFO] [1735961898.862520]: cleared current plan +[INFO] [1735961898.864757]: moving to home +[INFO] [1735961902.069248]: reached home +[INFO] [1735961902.071574]: reset action history +[INFO] [1735961905.048217]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/requirements.txt b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-metadata.json b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..afd3053b1e5735f6b96f653a4ded5dc011e2cf47 --- /dev/null +++ b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T03:34:29.719513", + "startedAt": "2025-01-04T03:34:29.334534", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3562.3318750000003, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4497.391, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.952, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.014, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 851.2897453308105 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..01680638e7f8869150f9cd65c51c31f119561bcf --- /dev/null +++ b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "fdd6e363a52ea071e0fdbd66633825c74224cd431444cb99cd1df4b7f4cb9382", "size": 733172, "path": "media/videos/eval/0_eval_0_fdd6e363a52ea071e0fd.mp4"}, "global_step": 609, "_timestamp": 1735961909.3944168, "_runtime": 240.04822492599487, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "2c428916b5dd1a4c5cc243e4e8fbcff643e248befe72c1208fec7e7d566c7b3a", "size": 1169557, "path": "media/videos/eval/1_eval_1_2c428916b5dd1a4c5cc2.mp4"}, "eval/num_success": 1.0, "episode": 30.0, "eval/success_rate": 0.20000000298023224, "eval/success_rate_lb_0.01": 0.02388051152229309, "eval/success_rate_ub_0.01": 0.7333634495735168, "eval/success_rate_lb_0.025": 0.015644066035747528, "eval/success_rate_ub_0.025": 0.688895583152771, "eval/success_rate_lb_0.05": 0.02994169294834137, "eval/success_rate_ub_0.05": 0.5159374475479126, "eval/success_rate_lb_0.1": 0.021239768713712692, "eval/success_rate_ub_0.1": 0.45248472690582275, "eval/2_eval": {"_type": "video-file", "sha256": "0a6d5b71a8cbdc8ee1f71bb0aacbebe162a8f77a2ae3ed3ef9cf25c14635e28d", "size": 1195260, "path": "media/videos/eval/2_eval_2_0a6d5b71a8cbdc8ee1f7.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "5052d6603d25fb3c55cc3cce560e628a3d0a21775a672e5ebee4b704b5efab39", "size": 1188321, "path": "media/videos/eval/3_eval_3_5052d6603d25fb3c55cc.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "db949b22d5fdd234cf63c7f23cb14cd34ce9b2ce201fd2be2271329eafcc658d", "size": 1229458, "path": "media/videos/eval/4_eval_4_db949b22d5fdd234cf63.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "2d447e7da5390d365153b465cab4c070d4dcc878bdbcba3e92cb6f4e90800649", "size": 405, "artifact_path": "wandb-client-artifact://nm1k5tf9ltzqlzk9nnv732e5b9up8xxn60rxrptujd2brxtocwcdvgnson7nhk6ducrksqljrsvccsdibzek39sjvb4i6pujv586npzurfr3pgezjy4lietq0actwoyw/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://1eo2ap2jyy9kfyk9omtzvot6hkyjmt2m719fleqphsrema7letun1be550tu0h4g1imreml3i5xkqapqbvy914kesjme4xkkfenu3aqs1qraiv81mizj5mcydkxzo0kh:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 23, "total_num_episodes": 30, "total_success_rate": 0.7666666666666667, "total_success_rate_lb_0.01": 0.579828977568873, "total_success_rate_ub_0.01": 0.9037904440549078, "total_success_rate_lb_0.025": 0.5848899036476107, "total_success_rate_ub_0.025": 0.8843533768481209, "total_success_rate_lb_0.05": 0.6088560074350085, "total_success_rate_ub_0.05": 0.8762526027105753, "total_success_rate_lb_0.1": 0.6711570508435782, "total_success_rate_ub_0.1": 0.858932282703854, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "5c9c8917af98e9d22a01309f067d41170270ac5fae676cb9720dff4a487fff1d", "size": 518, "artifact_path": "wandb-client-artifact://445wq8cfw0gdd9ht10kgu9j7q7kjktofyvsamu5ctae0a1c5wb0ri2vrhsz5nt3j6w2kryliuo4b21y0jbbos6m89p0y7spigk3mzi15mues4dirldgxbkiqybv0vtll/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://nbgzxy8ovevtdq16h7r0xprsmf7rs549w067lq42d4lda1kq0cqh8qdc9mcco5ya9wd6tv775waqisw97doqq931cp3uhrvpp18pnrlile0ao2hccwgo6xirxig9ekvs:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_5c9c8917af98e9d22a01.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 259}, "eval/total_success_rate": 0.7666666507720947, "eval/total_success_rate_lb_0.01": 0.5798289775848389, "eval/total_success_rate_ub_0.01": 0.9037904143333435, "eval/total_success_rate_lb_0.025": 0.5848898887634277, "eval/total_success_rate_ub_0.025": 0.8843533992767334, "eval/total_success_rate_lb_0.05": 0.6088560223579407, "eval/total_success_rate_ub_0.05": 0.8762525916099548, "eval/total_success_rate_lb_0.1": 0.6711570620536804, "eval/total_success_rate_ub_0.1": 0.8589322566986084} \ No newline at end of file diff --git a/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/logs/debug-internal.log b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ccb63a5e9c2b7010159e48a1549f925c66b8b731 --- /dev/null +++ b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/logs/debug-internal.log @@ -0,0 +1,757 @@ +2025-01-03 22:34:29,346 INFO StreamThr :1984755 [internal.py:wandb_internal():85] W&B internal server running at pid: 1984755, started at: 2025-01-03 22:34:29.345769 +2025-01-03 22:34:29,347 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:34:29,357 INFO WriterThread:1984755 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/run-qulr7kiu.wandb +2025-01-03 22:34:29,358 DEBUG SenderThread:1984755 [sender.py:send():379] send: header +2025-01-03 22:34:29,365 DEBUG SenderThread:1984755 [sender.py:send():379] send: run +2025-01-03 22:34:29,612 INFO SenderThread:1984755 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files +2025-01-03 22:34:29,612 INFO SenderThread:1984755 [sender.py:_start_run_threads():1188] run started: qulr7kiu with start time 1735961669.346192 +2025-01-03 22:34:29,629 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:34:29,629 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:34:29,700 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:34:29,705 DEBUG HandlerThread:1984755 [system_info.py:__init__():26] System info init +2025-01-03 22:34:29,705 DEBUG HandlerThread:1984755 [system_info.py:__init__():41] System info init done +2025-01-03 22:34:29,705 INFO HandlerThread:1984755 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:34:29,705 INFO SystemMonitor:1984755 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:34:29,706 INFO HandlerThread:1984755 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:34:29,706 INFO SystemMonitor:1984755 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:34:29,707 INFO SystemMonitor:1984755 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:34:29,708 INFO SystemMonitor:1984755 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:34:29,708 INFO SystemMonitor:1984755 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:34:29,709 INFO SystemMonitor:1984755 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:34:29,719 DEBUG HandlerThread:1984755 [system_info.py:probe():152] Probing system +2025-01-03 22:34:29,720 DEBUG HandlerThread:1984755 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:34:29,725 DEBUG HandlerThread:1984755 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:34:29,725 DEBUG HandlerThread:1984755 [system_info.py:probe():200] Probing system done +2025-01-03 22:34:29,725 DEBUG HandlerThread:1984755 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:34:29.719513', 'startedAt': '2025-01-04T03:34:29.334534', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.3318750000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4497.391, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.952, 'min': 800.0, 'max': 4700.0}, {'current': 4500.014, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 851.2897453308105}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:34:29,726 INFO HandlerThread:1984755 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:34:29,726 INFO HandlerThread:1984755 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:34:29,726 DEBUG HandlerThread:1984755 [system_info.py:_save_code():44] Saving code +2025-01-03 22:34:29,727 DEBUG HandlerThread:1984755 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:34:29,727 DEBUG HandlerThread:1984755 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:34:29,790 DEBUG HandlerThread:1984755 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:34:29,791 INFO HandlerThread:1984755 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:34:29,794 DEBUG SenderThread:1984755 [sender.py:send():379] send: files +2025-01-03 22:34:29,794 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:34:29,794 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:34:29,794 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:34:30,024 INFO wandb-upload_0:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/dypoc7d8-wandb-metadata.json +2025-01-03 22:34:30,029 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:34:30,029 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:34:30,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:30,031 DEBUG SenderThread:1984755 [sender.py:send():379] send: telemetry +2025-01-03 22:34:30,065 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/rfojwaap-diff.patch +2025-01-03 22:34:30,073 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:34:30,073 INFO HandlerThread:1984755 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/tb" + save: true +} +control { + relay_id: "265f69dd70ed47ba83a6b9d5dd2fa93c" +} +_info { + stream_id: "qulr7kiu" +} + +2025-01-03 22:34:30,073 DEBUG HandlerThread:1984755 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:34:30,074 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:34:30,232 DEBUG SenderThread:1984755 [sender.py:send():379] send: tbrecord +2025-01-03 22:34:30,233 DEBUG SenderThread:1984755 [sender.py:send():379] send: files +2025-01-03 22:34:30,233 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 with policy live +2025-01-03 22:34:30,451 INFO wandb-upload_0:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/7k8cpf6i-events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:34:30,613 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/code/FISH/eval_robot.py +2025-01-03 22:34:30,613 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-metadata.json +2025-01-03 22:34:30,614 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:34:30,614 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/requirements.txt +2025-01-03 22:34:30,614 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:34:30,614 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/diff.patch +2025-01-03 22:34:30,614 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/code +2025-01-03 22:34:30,614 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/code/FISH +2025-01-03 22:34:31,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:32,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:32,613 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:34:33,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:34,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:34,550 INFO wandb-upload_1:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/2r3f18u4-code/FISH/eval_robot.py +2025-01-03 22:34:35,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:34:35,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:36,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:36,926 DEBUG SenderThread:1984755 [sender.py:send():379] send: config +2025-01-03 22:34:37,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:38,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:38,615 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:34:39,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:40,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:34:40,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:41,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:42,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:42,616 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:34:43,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:44,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:45,029 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:34:45,030 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:34:45,032 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:45,189 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:34:46,033 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:46,617 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:34:47,032 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:48,033 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:48,618 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:34:49,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:50,033 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:50,618 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:34:51,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:34:51,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:52,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:52,619 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:34:53,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:54,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:55,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:56,031 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:34:56,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:57,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:58,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:34:59,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:00,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:35:00,030 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:35:00,073 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:01,035 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:02,035 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:03,035 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:04,035 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:04,689 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:05,035 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:05,622 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/config.yaml +2025-01-03 22:35:06,035 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:07,035 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:07,932 DEBUG SenderThread:1984755 [sender.py:send():379] send: files +2025-01-03 22:35:07,932 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_fdd6e363a52ea071e0fd.mp4 with policy now +2025-01-03 22:35:07,932 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:35:07,933 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:35:07,934 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:35:07,936 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:35:08,036 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:08,138 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/tk5x1qa7-media/videos/eval/0_eval_0_fdd6e363a52ea071e0fd.mp4 +2025-01-03 22:35:08,623 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:35:08,623 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:35:08,623 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/0_eval_0_fdd6e363a52ea071e0fd.mp4 +2025-01-03 22:35:08,623 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval +2025-01-03 22:35:08,623 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos +2025-01-03 22:35:08,624 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media +2025-01-03 22:35:08,870 INFO wandb-upload_1:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/62fkgfei-media/videos/eval/0_eval_0_fdd6e363a52ea071e0fd.mp4 +2025-01-03 22:35:09,039 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:09,953 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:10,037 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:10,623 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:35:11,037 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:12,037 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:12,624 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:35:13,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:14,038 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:15,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:35:15,030 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:35:15,073 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:15,093 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:16,038 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:16,625 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:35:17,039 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:18,039 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:19,039 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:20,039 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:21,032 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:21,039 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:22,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:23,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:24,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:25,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:26,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:27,032 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:27,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:28,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:29,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:29,709 DEBUG SystemMonitor:1984755 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:35:29,710 DEBUG SenderThread:1984755 [sender.py:send():379] send: stats +2025-01-03 22:35:30,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:35:30,030 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:35:30,040 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:30,630 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:35:30,630 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:35:30,991 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/910defue-events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:35:31,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:32,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:32,631 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:35:32,725 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:33,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:34,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:34,631 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:35:35,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:36,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:37,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:38,032 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:38,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:39,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:40,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:41,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:42,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:43,032 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:43,042 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:44,042 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:45,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:35:45,030 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:35:45,073 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:46,042 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:47,042 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:48,042 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:48,158 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:49,042 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:50,042 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:51,042 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:52,042 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:53,043 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:54,033 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:54,043 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:55,043 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:56,043 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:57,043 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:58,043 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:58,638 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:35:59,043 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:35:59,213 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:35:59,711 DEBUG SenderThread:1984755 [sender.py:send():379] send: stats +2025-01-03 22:36:00,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:36:00,030 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:36:00,077 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:01,044 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:01,678 DEBUG SenderThread:1984755 [sender.py:send():379] send: files +2025-01-03 22:36:01,678 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_2c428916b5dd1a4c5cc2.mp4 with policy now +2025-01-03 22:36:01,686 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:36:01,686 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:36:01,686 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:36:01,693 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:36:01,882 INFO wandb-upload_1:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/i692njwn-media/videos/eval/1_eval_1_2c428916b5dd1a4c5cc2.mp4 +2025-01-03 22:36:02,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:02,640 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:02,640 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:36:02,640 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/1_eval_1_2c428916b5dd1a4c5cc2.mp4 +2025-01-03 22:36:02,640 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval +2025-01-03 22:36:02,903 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/24jemtfr-media/videos/eval/1_eval_1_2c428916b5dd1a4c5cc2.mp4 +2025-01-03 22:36:03,044 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:04,045 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:04,640 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:04,721 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:05,049 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:06,047 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:06,641 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:07,049 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:08,053 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:09,045 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:10,037 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:10,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:10,642 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:11,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:12,045 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:13,045 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:13,643 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:36:14,045 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:14,465 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:36:14,465 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:36:14,466 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:36:14,644 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:14,644 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:36:15,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:36:15,030 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:36:15,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:15,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:16,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:16,644 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:17,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:18,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:18,645 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:19,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:20,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:21,038 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:21,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:22,046 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:23,049 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:24,049 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:25,049 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:26,049 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:27,037 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:27,049 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:28,051 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:29,052 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:29,711 DEBUG SenderThread:1984755 [sender.py:send():379] send: stats +2025-01-03 22:36:30,030 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:36:30,030 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:36:30,073 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:31,052 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:32,052 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:32,110 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:33,052 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:34,052 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:35,052 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:36,052 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:36,651 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:37,053 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:37,530 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:38,053 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:39,054 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:40,054 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:41,061 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:42,061 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:43,038 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:43,061 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:44,069 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:45,036 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:36:45,036 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:36:45,070 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:45,346 DEBUG SenderThread:1984755 [sender.py:send():379] send: files +2025-01-03 22:36:45,346 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_0a6d5b71a8cbdc8ee1f7.mp4 with policy now +2025-01-03 22:36:45,355 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:36:45,355 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:36:45,355 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:36:45,357 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:36:45,556 INFO wandb-upload_1:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/z15j1f3k-media/videos/eval/2_eval_2_0a6d5b71a8cbdc8ee1f7.mp4 +2025-01-03 22:36:45,658 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:36:45,658 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/2_eval_2_0a6d5b71a8cbdc8ee1f7.mp4 +2025-01-03 22:36:45,658 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval +2025-01-03 22:36:45,939 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/mh60bmdo-media/videos/eval/2_eval_2_0a6d5b71a8cbdc8ee1f7.mp4 +2025-01-03 22:36:46,070 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:46,658 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:47,071 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:48,069 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:48,385 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:49,072 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:50,072 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:50,660 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:51,072 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:52,071 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:52,660 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:36:53,074 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:53,829 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:54,072 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:55,074 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:56,072 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:57,074 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:58,075 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:59,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:36:59,077 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:36:59,711 DEBUG SenderThread:1984755 [sender.py:send():379] send: stats +2025-01-03 22:37:00,036 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:37:00,036 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:37:00,077 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:01,080 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:02,078 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:03,076 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:04,080 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:04,098 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:05,078 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:06,109 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:06,311 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:37:06,311 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:37:06,311 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:37:06,665 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:37:06,665 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:37:06,907 INFO wandb-upload_1:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/sz3ym9u1-events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:37:07,087 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:08,084 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:08,665 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:37:09,084 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:09,137 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:10,089 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:10,666 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:37:11,083 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:12,083 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:12,667 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:37:13,084 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:14,083 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:14,154 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:15,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:37:15,034 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:37:15,083 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:16,084 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:17,084 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:18,084 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:19,084 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:19,159 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:20,084 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:21,085 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:22,085 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:23,085 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:24,085 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:25,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:25,085 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:26,085 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:27,086 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:28,086 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:29,086 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:29,712 DEBUG SenderThread:1984755 [sender.py:send():379] send: stats +2025-01-03 22:37:30,034 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:37:30,034 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:37:30,086 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:30,157 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:31,086 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:32,086 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:33,086 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:34,086 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:35,086 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:36,041 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:36,087 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:37,088 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:38,092 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:38,145 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:37:38,145 DEBUG SenderThread:1984755 [sender.py:send():379] send: files +2025-01-03 22:37:38,146 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_5052d6603d25fb3c55cc.mp4 with policy now +2025-01-03 22:37:38,146 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:37:38,146 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:37:38,147 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:37:38,394 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/6g0cs19f-media/videos/eval/3_eval_3_5052d6603d25fb3c55cc.mp4 +2025-01-03 22:37:38,676 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:37:38,676 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:37:38,676 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/3_eval_3_5052d6603d25fb3c55cc.mp4 +2025-01-03 22:37:38,676 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval +2025-01-03 22:37:38,947 INFO wandb-upload_1:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/oind4h1b-media/videos/eval/3_eval_3_5052d6603d25fb3c55cc.mp4 +2025-01-03 22:37:39,096 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:40,093 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:40,676 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:37:41,047 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:41,095 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:42,098 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:42,677 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:37:43,101 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:44,107 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:45,050 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:37:45,050 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:37:45,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:46,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:46,146 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:46,678 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:37:46,679 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:37:47,115 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:47,359 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:37:47,360 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:37:47,360 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:37:47,679 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:37:47,679 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:37:48,115 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:48,679 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:37:49,110 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:50,113 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:51,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:51,955 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:52,110 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:52,680 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:37:53,110 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:54,110 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:55,110 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:56,110 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:57,045 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:37:57,110 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:58,110 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:59,111 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:37:59,713 DEBUG SenderThread:1984755 [sender.py:send():379] send: stats +2025-01-03 22:38:00,044 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:38:00,045 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:38:00,111 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:01,111 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:02,111 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:02,137 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:03,111 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:04,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:05,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:06,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:07,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:08,048 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:08,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:09,112 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:10,113 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:11,113 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:12,113 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:12,688 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:38:13,113 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:13,949 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:14,113 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:15,044 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:38:15,045 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:38:15,114 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:16,114 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:17,114 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:18,139 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:18,690 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:38:18,834 DEBUG SenderThread:1984755 [sender.py:send():379] send: files +2025-01-03 22:38:18,834 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_db949b22d5fdd234cf63.mp4 with policy now +2025-01-03 22:38:18,840 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:38:18,840 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:38:18,840 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:18,841 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:19,039 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/b1l0r5zs-media/videos/eval/4_eval_4_db949b22d5fdd234cf63.mp4 +2025-01-03 22:38:19,129 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:19,690 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:38:19,690 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/4_eval_4_db949b22d5fdd234cf63.mp4 +2025-01-03 22:38:19,690 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval +2025-01-03 22:38:19,865 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:19,957 INFO wandb-upload_1:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/yhytphbf-media/videos/eval/4_eval_4_db949b22d5fdd234cf63.mp4 +2025-01-03 22:38:20,131 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:20,690 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:38:21,131 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:22,131 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:23,130 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:24,127 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:24,692 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:38:25,051 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:25,128 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:26,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:26,692 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:38:27,145 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:28,138 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:28,406 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:38:28,407 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:28,407 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:28,693 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:38:28,693 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:38:28,833 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:38:28,833 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:38:28,924 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/bkk6vjv9-events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:38:29,101 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:38:29,132 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:29,326 INFO wandb-upload_1:1984755 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp7qpap0mi +2025-01-03 22:38:29,486 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:38:29,494 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:38:29,694 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:38:29,694 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/table/eval/total_success_rate_ci_6_5c9c8917af98e9d22a01.table.json +2025-01-03 22:38:29,694 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json +2025-01-03 22:38:29,694 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media +2025-01-03 22:38:29,694 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/table +2025-01-03 22:38:29,694 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/table/eval +2025-01-03 22:38:29,762 INFO SenderThread:1984755 [sender.py:send_request_log_artifact():1525] logged artifact run-qulr7kiu-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk4NDM2MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4OTIxMg==', 'latestArtifact': None}} +2025-01-03 22:38:29,762 DEBUG SenderThread:1984755 [sender.py:send():379] send: files +2025-01-03 22:38:29,762 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json with policy now +2025-01-03 22:38:29,762 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:38:29,763 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,763 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,763 DEBUG SenderThread:1984755 [sender.py:send():379] send: telemetry +2025-01-03 22:38:29,763 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,764 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,764 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,764 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,764 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,764 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,765 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,765 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,765 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,766 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,766 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,766 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,766 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,766 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,767 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,767 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,767 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,767 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,767 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,768 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,768 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:29,768 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:29,768 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:38:29,936 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/kgw2mw0a-media/table/eval/total_success_rate_ci_6_5c9c8917af98e9d22a01.table.json +2025-01-03 22:38:29,955 INFO wandb-upload_0:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/1y2cr3kt-media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json +2025-01-03 22:38:30,045 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:38:30,132 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:30,142 INFO wandb-upload_1:1984755 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpq3o80krz +2025-01-03 22:38:30,189 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/uh4fh8i0-media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json +2025-01-03 22:38:30,694 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:38:30,694 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:38:30,703 INFO SenderThread:1984755 [sender.py:send_request_log_artifact():1525] logged artifact run-qulr7kiu-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk4NDM2OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4OTIxNA==', 'latestArtifact': None}} +2025-01-03 22:38:30,703 DEBUG SenderThread:1984755 [sender.py:send():379] send: files +2025-01-03 22:38:30,703 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_5c9c8917af98e9d22a01.table.json with policy now +2025-01-03 22:38:30,703 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:38:30,703 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:30,704 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:30,704 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:30,704 DEBUG SenderThread:1984755 [sender.py:send():379] send: stats +2025-01-03 22:38:30,705 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:38:30,938 INFO wandb-upload_0:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/9mnrah13-media/table/eval/total_success_rate_ci_6_5c9c8917af98e9d22a01.table.json +2025-01-03 22:38:31,132 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:31,695 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:38:32,132 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:33,132 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:34,132 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:35,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:36,050 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:36,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:37,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:38,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:39,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:40,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:41,050 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:41,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:42,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:43,133 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:44,134 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:45,045 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:38:45,045 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:38:45,134 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:46,134 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:46,636 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:47,134 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:48,134 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:49,135 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:49,270 DEBUG SenderThread:1984755 [sender.py:send():379] send: exit +2025-01-03 22:38:49,270 INFO SenderThread:1984755 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 22:38:49,270 INFO SenderThread:1984755 [sender.py:send_exit():588] handling runtime: 259 +2025-01-03 22:38:49,271 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:49,271 INFO SenderThread:1984755 [sender.py:send_exit():594] send defer +2025-01-03 22:38:49,271 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:49,271 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 22:38:49,272 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:49,272 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 22:38:49,272 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 1 +2025-01-03 22:38:49,272 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:49,272 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 22:38:49,272 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:49,272 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 22:38:49,272 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 2 +2025-01-03 22:38:49,272 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:49,272 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 22:38:49,272 INFO HandlerThread:1984755 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 22:38:49,273 DEBUG SystemMonitor:1984755 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 22:38:49,273 DEBUG SystemMonitor:1984755 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 22:38:49,273 INFO HandlerThread:1984755 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 22:38:49,273 INFO HandlerThread:1984755 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 22:38:49,324 INFO HandlerThread:1984755 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 22:38:49,324 INFO HandlerThread:1984755 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 22:38:49,324 INFO HandlerThread:1984755 [interfaces.py:finish():200] Joined network monitor +2025-01-03 22:38:49,324 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:49,324 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 22:38:49,325 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 3 +2025-01-03 22:38:49,325 DEBUG SenderThread:1984755 [sender.py:send():379] send: stats +2025-01-03 22:38:49,325 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:49,325 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 22:38:49,325 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:49,325 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 22:38:49,325 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 4 +2025-01-03 22:38:49,325 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:49,325 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 22:38:49,702 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:38:50,702 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:38:52,703 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/config.yaml +2025-01-03 22:38:55,422 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:38:55,423 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:55,423 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:38:55,424 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:55,424 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 22:38:55,424 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 5 +2025-01-03 22:38:55,424 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:38:55,424 DEBUG SenderThread:1984755 [sender.py:send():379] send: history +2025-01-03 22:38:55,424 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:38:55,425 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:55,426 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 22:38:55,426 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:55,426 DEBUG SenderThread:1984755 [sender.py:send():379] send: summary +2025-01-03 22:38:55,430 INFO SenderThread:1984755 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:38:55,430 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:55,430 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 22:38:55,430 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 6 +2025-01-03 22:38:55,430 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:55,430 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 22:38:55,430 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:55,430 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 22:38:55,431 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 7 +2025-01-03 22:38:55,431 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:38:55,431 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:55,431 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 22:38:55,431 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:55,431 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 22:38:55,704 INFO Thread-12 :1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:38:56,271 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:38:57,869 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 8 +2025-01-03 22:38:57,869 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:38:57,869 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:57,869 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 22:38:57,870 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:57,870 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 22:38:57,870 INFO SenderThread:1984755 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 22:38:57,874 INFO SenderThread:1984755 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 22:38:57,876 INFO SenderThread:1984755 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 22:38:57,890 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 9 +2025-01-03 22:38:57,890 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:57,890 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 22:38:57,890 DEBUG SenderThread:1984755 [sender.py:send():379] send: artifact +2025-01-03 22:38:58,118 INFO SenderThread:1984755 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjYyMA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjYyMA==', 'versionIndex': 412}}} +2025-01-03 22:38:58,118 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:58,118 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 22:38:58,118 INFO SenderThread:1984755 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 22:38:58,272 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:38:58,706 INFO SenderThread:1984755 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:38:58,706 INFO SenderThread:1984755 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files +2025-01-03 22:38:58,706 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/config.yaml config.yaml +2025-01-03 22:38:58,706 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json wandb-summary.json +2025-01-03 22:38:58,708 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/requirements.txt requirements.txt +2025-01-03 22:38:58,708 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-metadata.json wandb-metadata.json +2025-01-03 22:38:58,708 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:38:58,711 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log output.log +2025-01-03 22:38:58,712 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/diff.patch diff.patch +2025-01-03 22:38:58,712 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 22:38:58,712 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/2_eval_2_0a6d5b71a8cbdc8ee1f7.mp4 media/videos/eval/2_eval_2_0a6d5b71a8cbdc8ee1f7.mp4 +2025-01-03 22:38:58,712 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/4_eval_4_db949b22d5fdd234cf63.mp4 media/videos/eval/4_eval_4_db949b22d5fdd234cf63.mp4 +2025-01-03 22:38:58,712 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/0_eval_0_fdd6e363a52ea071e0fd.mp4 media/videos/eval/0_eval_0_fdd6e363a52ea071e0fd.mp4 +2025-01-03 22:38:58,712 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/3_eval_3_5052d6603d25fb3c55cc.mp4 media/videos/eval/3_eval_3_5052d6603d25fb3c55cc.mp4 +2025-01-03 22:38:58,712 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/videos/eval/1_eval_1_2c428916b5dd1a4c5cc2.mp4 media/videos/eval/1_eval_1_2c428916b5dd1a4c5cc2.mp4 +2025-01-03 22:38:58,712 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json +2025-01-03 22:38:58,713 INFO SenderThread:1984755 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/media/table/eval/total_success_rate_ci_6_5c9c8917af98e9d22a01.table.json media/table/eval/total_success_rate_ci_6_5c9c8917af98e9d22a01.table.json +2025-01-03 22:38:58,713 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 10 +2025-01-03 22:38:58,713 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:38:58,713 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:58,714 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 22:38:58,715 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:58,715 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 22:38:58,716 INFO SenderThread:1984755 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:38:58,878 INFO wandb-upload_0:1984755 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/wandb-summary.json +2025-01-03 22:38:58,889 INFO wandb-upload_2:1984755 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/config.yaml +2025-01-03 22:38:58,978 INFO wandb-upload_1:1984755 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/requirements.txt +2025-01-03 22:38:58,996 INFO wandb-upload_3:1984755 [upload_job.py:push():130] Uploaded file /tmp/tmpkdcm5melwandb/bw9txxql-events.out.tfevents.1735961670.leonmkim-ROG-Strix-G15CS-G15CS.1984590.0 +2025-01-03 22:38:58,998 INFO wandb-upload_4:1984755 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/files/output.log +2025-01-03 22:38:59,198 INFO Thread-11 (_thread_body):1984755 [sender.py:transition_state():614] send defer: 11 +2025-01-03 22:38:59,198 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:59,198 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 22:38:59,199 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:59,199 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 22:38:59,199 INFO SenderThread:1984755 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 22:38:59,200 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 12 +2025-01-03 22:38:59,200 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:59,200 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 22:38:59,200 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:59,201 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 22:38:59,201 INFO SenderThread:1984755 [file_stream.py:finish():601] file stream finish called +2025-01-03 22:38:59,272 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:38:59,390 INFO SenderThread:1984755 [file_stream.py:finish():605] file stream finish is done +2025-01-03 22:38:59,390 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 13 +2025-01-03 22:38:59,390 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:38:59,390 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:59,390 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 22:38:59,390 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:59,391 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 22:38:59,391 INFO SenderThread:1984755 [sender.py:transition_state():614] send defer: 14 +2025-01-03 22:38:59,391 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:38:59,391 INFO HandlerThread:1984755 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 22:38:59,391 DEBUG SenderThread:1984755 [sender.py:send():379] send: final +2025-01-03 22:38:59,391 DEBUG SenderThread:1984755 [sender.py:send():379] send: footer +2025-01-03 22:38:59,391 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: defer +2025-01-03 22:38:59,391 INFO SenderThread:1984755 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 22:38:59,391 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:38:59,392 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:38:59,392 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 22:38:59,392 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:38:59,392 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 22:38:59,393 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 22:38:59,393 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:38:59,393 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: server_info +2025-01-03 22:38:59,457 DEBUG SenderThread:1984755 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:38:59,458 INFO MainThread:1984755 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 22:38:59,458 INFO MainThread:1984755 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 22:38:59,458 INFO MainThread:1984755 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 22:38:59,458 DEBUG HandlerThread:1984755 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 22:38:59,458 INFO HandlerThread:1984755 [handler.py:finish():882] shutting down handler +2025-01-03 22:39:00,393 INFO WriterThread:1984755 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/run-qulr7kiu.wandb +2025-01-03 22:39:00,458 INFO SenderThread:1984755 [sender.py:finish():1615] shutting down sender +2025-01-03 22:39:00,458 INFO SenderThread:1984755 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:39:00,458 INFO SenderThread:1984755 [file_pusher.py:join():175] waiting for file pusher diff --git a/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/logs/debug.log b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..5340f2ac9cc9f8d36727cb8004a48ff88df697c5 --- /dev/null +++ b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:34:29,335 INFO MainThread:1984590 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:34:29,335 INFO MainThread:1984590 [wandb_setup.py:_flush():76] Configure stats pid to 1984590 +2025-01-03 22:34:29,335 INFO MainThread:1984590 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/settings +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/logs/debug.log +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/logs/debug-internal.log +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1922_1', 'wandb_notes': '1922_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu'} +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_init.py:init():619] starting backend +2025-01-03 22:34:29,336 INFO MainThread:1984590 [wandb_init.py:init():623] setting up manager +2025-01-03 22:34:29,344 INFO MainThread:1984590 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:34:29,345 INFO MainThread:1984590 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:34:29,357 INFO MainThread:1984590 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:34:29,363 INFO MainThread:1984590 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:34:29,629 INFO MainThread:1984590 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:34:29,694 INFO MainThread:1984590 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:34:29,695 INFO MainThread:1984590 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:34:30,030 INFO MainThread:1984590 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:34:30,030 INFO MainThread:1984590 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:34:30,030 INFO MainThread:1984590 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:34:30,030 INFO MainThread:1984590 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:34:30,032 INFO MainThread:1984590 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:34:30,032 INFO MainThread:1984590 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1922_1/qulr7kiu/tb, True +2025-01-03 22:34:36,926 INFO MainThread:1984590 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-03 22:39:00,459 WARNING MsgRouterThr:1984590 [router.py:message_loop():77] message_loop has been closed diff --git a/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/run-qulr7kiu.wandb b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/run-qulr7kiu.wandb new file mode 100644 index 0000000000000000000000000000000000000000..14fbb855e09060e464dda0c619a457051827db10 Binary files /dev/null and b/qulr7kiu/wandb/run-20250103_223429-qulr7kiu/run-qulr7kiu.wandb differ diff --git a/snapshot_12432.pt b/snapshot_12432.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3defa2e413fe0ad491f4d07cad63db25202d6b7 --- /dev/null +++ b/snapshot_12432.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d9c892236e49099a58d4ab1a95df829e821ed0d92e33e8c69bb9534890e6901 +size 910102934 diff --git a/xt2ne5el/episode_rosbags/episode_1_2025-01-04-22-53-46.bag b/xt2ne5el/episode_rosbags/episode_1_2025-01-04-22-53-46.bag new file mode 100644 index 0000000000000000000000000000000000000000..95cc62657b306d393adc923f2ebf9b77cfae5b62 --- /dev/null +++ b/xt2ne5el/episode_rosbags/episode_1_2025-01-04-22-53-46.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd5079eae7d3efbc9d3e0ee0e6148811a430f1ccb054b87fb920145b4b948020 +size 1411980342 diff --git a/xt2ne5el/episode_rosbags/episode_2_2025-01-04-22-54-27.bag b/xt2ne5el/episode_rosbags/episode_2_2025-01-04-22-54-27.bag new file mode 100644 index 0000000000000000000000000000000000000000..ae2c496f66289bfdb92705afd629a8d538d656a3 --- /dev/null +++ b/xt2ne5el/episode_rosbags/episode_2_2025-01-04-22-54-27.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:317258389db53792853a1e87465c25b8cce4a35ac0385249dd2ee0b99c84922d +size 1413719745 diff --git a/xt2ne5el/episode_rosbags/episode_3_2025-01-04-22-55-20.bag b/xt2ne5el/episode_rosbags/episode_3_2025-01-04-22-55-20.bag new file mode 100644 index 0000000000000000000000000000000000000000..bae4ef4bcd5659172753fb41f7b1c5f470946188 --- /dev/null +++ b/xt2ne5el/episode_rosbags/episode_3_2025-01-04-22-55-20.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fcbbbe210a7e00724c1c05edf7d1157c2b15fa0a7d325e5cbb1c51c50eabdc7 +size 1411094144