diff --git a/.gitattributes b/.gitattributes index 4fb0f5e1cdf36b14bdb85568738a42b7ada17a23..5fdd73aaa3460f044ebba3ffad2cf0d594de9e88 100644 --- a/.gitattributes +++ b/.gitattributes @@ -83,3 +83,41 @@ up4gjipz/wandb/run-20250106_200056-up4gjipz/files/media/videos/eval/2_eval_2_2af up4gjipz/wandb/run-20250106_200056-up4gjipz/files/media/videos/eval/3_eval_3_3f13de8c1cf058ceccfd.mp4 filter=lfs diff=lfs merge=lfs -text kkp1x4wb/episode_rosbags/episode_0_2025-01-06-18-13-37.bag filter=lfs diff=lfs merge=lfs -text up4gjipz/wandb/run-20250106_200056-up4gjipz/files/media/videos/eval/0_eval_0_10ebed5c632009219ad7.mp4 filter=lfs diff=lfs merge=lfs -text +up4gjipz/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up4gjipz/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up4gjipz/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up4gjipz/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up4gjipz/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kkp1x4wb/episode_rosbags/episode_2_2025-01-06-18-18-23.bag filter=lfs diff=lfs merge=lfs -text +3fs5lhth/episode_rosbags/episode_1_2025-01-06-22-58-42.bag filter=lfs diff=lfs merge=lfs -text +3fs5lhth/episode_rosbags/episode_2_2025-01-06-22-59-30.bag filter=lfs diff=lfs merge=lfs -text +kd9mh2vh/episode_rosbags/episode_2_2025-01-07-16-24-51.bag filter=lfs diff=lfs merge=lfs -text +x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/3_eval_3_88082439ff7f6a045e2d.mp4 filter=lfs diff=lfs merge=lfs -text +x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/0_eval_0_7ad56c694ef7147e5404.mp4 filter=lfs diff=lfs merge=lfs -text +x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/1_eval_1_c12723790e5302e09cec.mp4 filter=lfs diff=lfs merge=lfs -text +x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/4_eval_4_df2f032446cb0a313989.mp4 filter=lfs diff=lfs merge=lfs -text +x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/2_eval_2_85ba534ec05635d71996.mp4 filter=lfs diff=lfs merge=lfs -text +x85ya2ao/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +x85ya2ao/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +x85ya2ao/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +x85ya2ao/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +x85ya2ao/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kd9mh2vh/episode_rosbags/episode_3_2025-01-07-16-25-41.bag filter=lfs diff=lfs merge=lfs -text +urh96fgp/episode_rosbags/episode_1_2025-01-07-20-12-33.bag filter=lfs diff=lfs merge=lfs -text +kkp1x4wb/episode_rosbags/episode_3_2025-01-06-18-19-20.bag filter=lfs diff=lfs merge=lfs -text +urh96fgp/episode_rosbags/episode_0_2025-01-07-20-11-41.bag filter=lfs diff=lfs merge=lfs -text +3fs5lhth/episode_rosbags/episode_0_2025-01-06-22-57-46.bag filter=lfs diff=lfs merge=lfs -text +m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/0_eval_0_22bed2761bf0b211c58d.mp4 filter=lfs diff=lfs merge=lfs -text +m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/3_eval_3_fd7b6d2ce3d8ed8b882d.mp4 filter=lfs diff=lfs merge=lfs -text +m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/2_eval_2_e403f7a04c6fe992037c.mp4 filter=lfs diff=lfs merge=lfs -text +m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/4_eval_4_7504f2a27cb28e455eb2.mp4 filter=lfs diff=lfs merge=lfs -text +m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/1_eval_1_ef07a287a59b234dfd50.mp4 filter=lfs diff=lfs merge=lfs -text +m386gyxa/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +m386gyxa/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +m386gyxa/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up4gjipz/episode_rosbags/episode_3_2025-01-06-20-03-53.bag filter=lfs diff=lfs merge=lfs -text +m386gyxa/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kd9mh2vh/episode_rosbags/episode_4_2025-01-07-16-26-45.bag filter=lfs diff=lfs merge=lfs -text +m386gyxa/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kd9mh2vh/episode_rosbags/episode_1_2025-01-07-16-23-54.bag filter=lfs diff=lfs merge=lfs -text +kd9mh2vh/episode_rosbags/episode_0_2025-01-07-16-22-51.bag filter=lfs diff=lfs merge=lfs -text diff --git a/3fs5lhth/episode_rosbags/episode_0_2025-01-06-22-57-46.bag b/3fs5lhth/episode_rosbags/episode_0_2025-01-06-22-57-46.bag new file mode 100644 index 0000000000000000000000000000000000000000..7c31f7ef4c7e50a2f48eac8f75860081585a8645 --- /dev/null +++ b/3fs5lhth/episode_rosbags/episode_0_2025-01-06-22-57-46.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c89fd02424eebe0fc5f34f006dada7373583496349030fe6f7372d1c69dca881 +size 2607509690 diff --git a/3fs5lhth/episode_rosbags/episode_1_2025-01-06-22-58-42.bag b/3fs5lhth/episode_rosbags/episode_1_2025-01-06-22-58-42.bag new file mode 100644 index 0000000000000000000000000000000000000000..19dc0c763eb26abc724fed40709983dbf1b45af1 --- /dev/null +++ b/3fs5lhth/episode_rosbags/episode_1_2025-01-06-22-58-42.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec8f9591146b30045e4044cbf323781ac44b0fcc7a899098fe842aca6bbc17ec +size 2601497288 diff --git a/3fs5lhth/episode_rosbags/episode_2_2025-01-06-22-59-30.bag b/3fs5lhth/episode_rosbags/episode_2_2025-01-06-22-59-30.bag new file mode 100644 index 0000000000000000000000000000000000000000..88ffd3f70c9a8c5bfa8041962a3d5c5532b778e0 --- /dev/null +++ b/3fs5lhth/episode_rosbags/episode_2_2025-01-06-22-59-30.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69335e08aa961cc6d1d58251188c0b22f942185591b84131de2068e294551c29 +size 2606005528 diff --git a/6tsw6mm6/.hydra/config.yaml b/6tsw6mm6/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd5530cb9a8f25d6d55135607d04c2061b4351d1 --- /dev/null +++ b/6tsw6mm6/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1017_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/6tsw6mm6/.hydra/hydra.yaml b/6tsw6mm6/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b180f95bd8dbe415436580b1f52826ac066b7a9e --- /dev/null +++ b/6tsw6mm6/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/6tsw6mm6 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/6tsw6mm6/.hydra/overrides.yaml b/6tsw6mm6/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/6tsw6mm6/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/6tsw6mm6/eval_policy.log b/6tsw6mm6/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..563eadba58285c57cb935f865313f78c55baeede --- /dev/null +++ b/6tsw6mm6/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-06 22:52:02,860][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 22:52:02,863][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-06 22:52:06,872][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/kd9mh2vh/episode_rosbags/episode_0_2025-01-07-16-22-51.bag b/kd9mh2vh/episode_rosbags/episode_0_2025-01-07-16-22-51.bag new file mode 100644 index 0000000000000000000000000000000000000000..6543f0c6a2f9279847e527cb3560c916aab62a7b --- /dev/null +++ b/kd9mh2vh/episode_rosbags/episode_0_2025-01-07-16-22-51.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae9431f6bcd78ea51098d3c57c5e3dce582115f086b7788e1fd6f02cc460352e +size 2605020228 diff --git a/kd9mh2vh/episode_rosbags/episode_1_2025-01-07-16-23-54.bag b/kd9mh2vh/episode_rosbags/episode_1_2025-01-07-16-23-54.bag new file mode 100644 index 0000000000000000000000000000000000000000..1bc183ea873de57179cbbfeb24e09cbb7bdc5cbb --- /dev/null +++ b/kd9mh2vh/episode_rosbags/episode_1_2025-01-07-16-23-54.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa36532a12161376ca0ad635451d829852f5f1457411f0dc224fa70a6243ed3a +size 2605164347 diff --git a/kd9mh2vh/episode_rosbags/episode_2_2025-01-07-16-24-51.bag b/kd9mh2vh/episode_rosbags/episode_2_2025-01-07-16-24-51.bag new file mode 100644 index 0000000000000000000000000000000000000000..f50c89678611e5337c8f3cdc42654df12f99e0f7 --- /dev/null +++ b/kd9mh2vh/episode_rosbags/episode_2_2025-01-07-16-24-51.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9430f8415cadfc85aebac334ab714cbef8e8879e47eca8e547c16330d4b86d00 +size 2603427128 diff --git a/kd9mh2vh/episode_rosbags/episode_3_2025-01-07-16-25-41.bag b/kd9mh2vh/episode_rosbags/episode_3_2025-01-07-16-25-41.bag new file mode 100644 index 0000000000000000000000000000000000000000..aad801024da04c18821d92b85ee2d721cf1688c5 --- /dev/null +++ b/kd9mh2vh/episode_rosbags/episode_3_2025-01-07-16-25-41.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:deea93ecaf6b67eaee8cb882fcf76fbcbed0598f98a5aa3f240de0b5fdecf122 +size 2603643392 diff --git a/kd9mh2vh/episode_rosbags/episode_4_2025-01-07-16-26-45.bag b/kd9mh2vh/episode_rosbags/episode_4_2025-01-07-16-26-45.bag new file mode 100644 index 0000000000000000000000000000000000000000..f3b88c842fd730012f68abb650b3bada7eb171e0 --- /dev/null +++ b/kd9mh2vh/episode_rosbags/episode_4_2025-01-07-16-26-45.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b8daab8382a90e8dcddcc7ed16afdc06153d75e1600979bb807d73d24255acf +size 2605905515 diff --git a/kkp1x4wb/episode_rosbags/episode_2_2025-01-06-18-18-23.bag b/kkp1x4wb/episode_rosbags/episode_2_2025-01-06-18-18-23.bag new file mode 100644 index 0000000000000000000000000000000000000000..f4599417f69cec5d354a208283bfd58d07f1c5ab --- /dev/null +++ b/kkp1x4wb/episode_rosbags/episode_2_2025-01-06-18-18-23.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e0d2d11642a7ba21ae6e4d304778518b0dbdf6364f09fe0edb2f9380ceb9dbe +size 2615290361 diff --git a/kkp1x4wb/episode_rosbags/episode_3_2025-01-06-18-19-20.bag b/kkp1x4wb/episode_rosbags/episode_3_2025-01-06-18-19-20.bag new file mode 100644 index 0000000000000000000000000000000000000000..e225bb9849e275d0596a9fb3dfe38ec91ec669fc --- /dev/null +++ b/kkp1x4wb/episode_rosbags/episode_3_2025-01-06-18-19-20.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc253474a462d9cb8c6371ee5bd62fb7bf9860c20c76d5061cc8e3852eeb9e82 +size 2608554400 diff --git a/m386gyxa/.hydra/config.yaml b/m386gyxa/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd5530cb9a8f25d6d55135607d04c2061b4351d1 --- /dev/null +++ b/m386gyxa/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1017_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/m386gyxa/.hydra/hydra.yaml b/m386gyxa/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..698388fd498f74f37e64684c9b350586f6c603d2 --- /dev/null +++ b/m386gyxa/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/m386gyxa/.hydra/overrides.yaml b/m386gyxa/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/m386gyxa/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/m386gyxa/episode_rosbags/aligned_depth_to_color_K.npy b/m386gyxa/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/m386gyxa/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/m386gyxa/episode_rosbags/cam_tf_world.npy b/m386gyxa/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/m386gyxa/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/m386gyxa/episode_rosbags/color_K.npy b/m386gyxa/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/m386gyxa/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/m386gyxa/episode_rosbags/depth_K.npy b/m386gyxa/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/m386gyxa/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/m386gyxa/eval_robot.log b/m386gyxa/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..bb60d47ea89434b37c95a90fe6ac7d0f4218cfe4 --- /dev/null +++ b/m386gyxa/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 20:05:51,084][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 20:05:51,086][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/m386gyxa/eval_video/0_eval.mp4 b/m386gyxa/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0d2ddc438a72922b2b81ebdb56d79a7cf651562d --- /dev/null +++ b/m386gyxa/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22bed2761bf0b211c58d04b8679118dbf8826eb7892546c26f648b5b459f41c8 +size 1459978 diff --git a/m386gyxa/eval_video/1_eval.mp4 b/m386gyxa/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7fd46c2cd325290dfd47f5eea99d6b64a31d17e1 --- /dev/null +++ b/m386gyxa/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef07a287a59b234dfd5075a2a428c07672ce90dc268a4c8ca8b99b816cd5387c +size 1455390 diff --git a/m386gyxa/eval_video/2_eval.mp4 b/m386gyxa/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..17944b2cffd0612103dbef1b60df3367b5043a0d --- /dev/null +++ b/m386gyxa/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e403f7a04c6fe992037c52f56c3716cf384e15d5c22c3529b37c30eab8242766 +size 1450880 diff --git a/m386gyxa/eval_video/3_eval.mp4 b/m386gyxa/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a7ab7d1bcf3b66a3b0a6c2a5690dcb62b9c0ef31 --- /dev/null +++ b/m386gyxa/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd7b6d2ce3d8ed8b882db063488e1aae0a7bc7de648f5503db03033e05b6e9ac +size 1462155 diff --git a/m386gyxa/eval_video/4_eval.mp4 b/m386gyxa/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df0de8ec97964bc98f8320f5fb5f6931cb130eeb --- /dev/null +++ b/m386gyxa/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7504f2a27cb28e455eb230f84c964216f73462455eb50dd74e4d6071cc9566c8 +size 1474668 diff --git a/m386gyxa/tb/events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 b/m386gyxa/tb/events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 new file mode 100644 index 0000000000000000000000000000000000000000..9c77ec122f0a6e9a2ef523745869083876f22aca --- /dev/null +++ b/m386gyxa/tb/events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48436ef1b4290e1b58d987f9900f8ed4927b58c7c1af75bd1ec4a42d3e027bd0 +size 6539 diff --git a/m386gyxa/wandb/debug-internal.log b/m386gyxa/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4f079ea180eb3d516dbd6e5c9d542e900e8eb337 --- /dev/null +++ b/m386gyxa/wandb/debug-internal.log @@ -0,0 +1,781 @@ +2025-01-06 20:05:57,045 INFO StreamThr :2104256 [internal.py:wandb_internal():85] W&B internal server running at pid: 2104256, started at: 2025-01-06 20:05:57.044943 +2025-01-06 20:05:57,046 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status +2025-01-06 20:05:57,060 INFO WriterThread:2104256 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/run-m386gyxa.wandb +2025-01-06 20:05:57,060 DEBUG SenderThread:2104256 [sender.py:send():379] send: header +2025-01-06 20:05:57,069 DEBUG SenderThread:2104256 [sender.py:send():379] send: run +2025-01-06 20:05:57,294 INFO SenderThread:2104256 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files +2025-01-06 20:05:57,295 INFO SenderThread:2104256 [sender.py:_start_run_threads():1188] run started: m386gyxa with start time 1736211957.045467 +2025-01-06 20:05:57,309 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 20:05:57,309 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: check_version +2025-01-06 20:05:57,354 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 20:05:57,359 DEBUG HandlerThread:2104256 [system_info.py:__init__():26] System info init +2025-01-06 20:05:57,359 DEBUG HandlerThread:2104256 [system_info.py:__init__():41] System info init done +2025-01-06 20:05:57,359 INFO HandlerThread:2104256 [system_monitor.py:start():194] Starting system monitor +2025-01-06 20:05:57,360 INFO SystemMonitor:2104256 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 20:05:57,360 INFO HandlerThread:2104256 [system_monitor.py:probe():214] Collecting system info +2025-01-06 20:05:57,360 INFO SystemMonitor:2104256 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 20:05:57,361 INFO SystemMonitor:2104256 [interfaces.py:start():188] Started disk monitoring +2025-01-06 20:05:57,361 INFO SystemMonitor:2104256 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 20:05:57,362 INFO SystemMonitor:2104256 [interfaces.py:start():188] Started memory monitoring +2025-01-06 20:05:57,362 INFO SystemMonitor:2104256 [interfaces.py:start():188] Started network monitoring +2025-01-06 20:05:57,372 DEBUG HandlerThread:2104256 [system_info.py:probe():152] Probing system +2025-01-06 20:05:57,373 DEBUG HandlerThread:2104256 [system_info.py:_probe_git():137] Probing git +2025-01-06 20:05:57,378 DEBUG HandlerThread:2104256 [system_info.py:_probe_git():145] Probing git done +2025-01-06 20:05:57,378 DEBUG HandlerThread:2104256 [system_info.py:probe():200] Probing system done +2025-01-06 20:05:57,378 DEBUG HandlerThread:2104256 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T01:05:57.372330', 'startedAt': '2025-01-07T01:05:57.037280', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.1145, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.859, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 438.2542266845703}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 20:05:57,378 INFO HandlerThread:2104256 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 20:05:57,378 INFO HandlerThread:2104256 [system_monitor.py:probe():227] Publishing system info +2025-01-06 20:05:57,378 DEBUG HandlerThread:2104256 [system_info.py:_save_code():44] Saving code +2025-01-06 20:05:57,379 DEBUG HandlerThread:2104256 [system_info.py:_save_code():67] Saving code done +2025-01-06 20:05:57,380 DEBUG HandlerThread:2104256 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 20:05:57,447 DEBUG HandlerThread:2104256 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 20:05:57,448 INFO HandlerThread:2104256 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 20:05:57,451 DEBUG SenderThread:2104256 [sender.py:send():379] send: files +2025-01-06 20:05:57,451 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 20:05:57,451 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 20:05:57,452 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 20:05:57,700 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 20:05:57,700 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: python_packages +2025-01-06 20:05:57,700 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:05:57,701 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:05:57,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:05:57,703 INFO HandlerThread:2104256 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/tb" + save: true +} +control { + relay_id: "e67a100e33da4f33931342645f4e8e07" +} +_info { + stream_id: "m386gyxa" +} + +2025-01-06 20:05:57,703 DEBUG HandlerThread:2104256 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 20:05:57,712 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/u85604lb-wandb-metadata.json +2025-01-06 20:05:57,732 INFO wandb-upload_1:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/uv7lvji5-code/FISH/eval_robot.py +2025-01-06 20:05:57,753 INFO wandb-upload_2:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/jhao2ns6-diff.patch +2025-01-06 20:05:57,766 DEBUG SenderThread:2104256 [sender.py:send():379] send: telemetry +2025-01-06 20:05:57,766 DEBUG SenderThread:2104256 [sender.py:send():379] send: tbrecord +2025-01-06 20:05:57,767 DEBUG SenderThread:2104256 [sender.py:send():379] send: files +2025-01-06 20:05:57,767 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 with policy live +2025-01-06 20:05:57,987 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/mplqfvx3-events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:05:58,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/requirements.txt +2025-01-06 20:05:58,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/diff.patch +2025-01-06 20:05:58,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-metadata.json +2025-01-06 20:05:58,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/code/FISH/eval_robot.py +2025-01-06 20:05:58,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:05:58,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:05:58,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/code/FISH +2025-01-06 20:05:58,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/code +2025-01-06 20:05:58,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:05:59,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:00,296 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:00,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:01,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:02,408 DEBUG SenderThread:2104256 [sender.py:send():379] send: config +2025-01-06 20:06:02,409 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:02,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:03,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:04,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:04,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:05,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:06,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:06,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:07,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:07,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:08,298 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:08,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:09,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:10,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:11,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:12,299 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:12,700 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:06:12,700 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:06:12,741 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:12,799 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:13,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:14,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:15,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:16,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:17,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:18,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:18,728 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:19,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:20,301 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:20,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:21,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:22,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:23,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:24,302 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:24,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:24,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:25,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:26,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:27,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:06:27,701 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:06:27,741 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:28,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:29,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:29,826 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:30,304 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/config.yaml +2025-01-06 20:06:30,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:31,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:32,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:33,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:34,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:35,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:35,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:36,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:37,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:38,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:39,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:40,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:40,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:41,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:42,700 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:06:42,701 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:06:42,741 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:43,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:44,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:45,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:45,753 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:46,308 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:46,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:47,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:48,309 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:48,707 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:49,707 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:50,707 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:51,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:51,707 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:52,708 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:53,709 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:54,711 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:55,710 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:56,710 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:57,170 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:57,362 DEBUG SystemMonitor:2104256 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 20:06:57,363 DEBUG SenderThread:2104256 [sender.py:send():379] send: stats +2025-01-06 20:06:57,372 DEBUG SenderThread:2104256 [sender.py:send():379] send: files +2025-01-06 20:06:57,372 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_22bed2761bf0b211c58d.mp4 with policy now +2025-01-06 20:06:57,372 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:06:57,374 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:06:57,374 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:06:57,374 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:06:57,674 INFO wandb-upload_2:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/9zghjcpi-media/videos/eval/0_eval_0_22bed2761bf0b211c58d.mp4 +2025-01-06 20:06:57,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:06:57,702 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:06:57,710 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:58,311 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:58,312 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/0_eval_0_22bed2761bf0b211c58d.mp4 +2025-01-06 20:06:58,312 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:06:58,312 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media +2025-01-06 20:06:58,312 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval +2025-01-06 20:06:58,312 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos +2025-01-06 20:06:58,630 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/93kmdg7n-media/videos/eval/0_eval_0_22bed2761bf0b211c58d.mp4 +2025-01-06 20:06:58,710 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:59,710 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:00,312 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:00,710 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:01,711 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:02,313 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:02,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:02,711 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:03,745 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:04,711 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:05,711 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:06,314 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:06,314 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:07:06,571 INFO wandb-upload_2:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/k2gcvpd2-events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:07:06,711 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:07,711 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:08,315 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:08,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:08,712 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:09,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:10,717 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:11,717 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:12,316 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:12,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:07:12,702 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:07:12,745 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:13,717 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:13,758 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:14,718 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:15,718 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:16,718 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:17,718 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:18,718 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:19,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:19,718 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:20,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:21,719 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:22,719 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:23,719 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:24,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:24,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:25,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:26,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:27,364 DEBUG SenderThread:2104256 [sender.py:send():379] send: stats +2025-01-06 20:07:27,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:07:27,702 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:07:27,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:28,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:29,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:30,322 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:30,653 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:30,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:31,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:32,323 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:32,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:33,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:34,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:35,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:35,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:36,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:37,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:38,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:39,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:40,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:40,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:41,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:42,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:07:42,701 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:07:42,745 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:43,722 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:44,722 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:45,722 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:45,829 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:46,006 DEBUG SenderThread:2104256 [sender.py:send():379] send: files +2025-01-06 20:07:46,006 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_ef07a287a59b234dfd50.mp4 with policy now +2025-01-06 20:07:46,006 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:07:46,007 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:07:46,007 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:07:46,012 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:07:46,286 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/dwi7bziw-media/videos/eval/1_eval_1_ef07a287a59b234dfd50.mp4 +2025-01-06 20:07:46,333 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:07:46,334 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:46,334 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/1_eval_1_ef07a287a59b234dfd50.mp4 +2025-01-06 20:07:46,334 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval +2025-01-06 20:07:46,641 INFO wandb-upload_2:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/tmycjluj-media/videos/eval/1_eval_1_ef07a287a59b234dfd50.mp4 +2025-01-06 20:07:46,726 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:47,731 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:48,334 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:48,731 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:49,769 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:50,732 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:51,731 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:51,769 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:52,335 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:52,731 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:53,732 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:54,336 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:54,732 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:55,113 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:07:55,114 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:07:55,114 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:07:55,337 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:07:55,337 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:07:55,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:56,337 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:56,732 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:57,364 DEBUG SenderThread:2104256 [sender.py:send():379] send: stats +2025-01-06 20:07:57,365 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:57,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:07:57,702 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:07:57,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:58,338 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:58,732 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:59,732 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:00,338 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:00,732 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:01,732 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:02,675 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:02,733 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:03,733 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:04,733 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:05,733 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:06,733 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:07,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:07,733 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:08,341 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:08,733 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:09,734 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:10,342 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:10,734 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:11,734 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:12,342 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:12,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:08:12,702 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:08:12,745 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:12,776 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:13,734 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:14,343 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:14,734 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:15,734 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:16,734 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:17,734 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:18,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:18,735 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:19,735 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:20,735 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:21,735 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:22,735 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:23,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:23,735 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:24,735 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:25,735 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:26,736 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:27,365 DEBUG SenderThread:2104256 [sender.py:send():379] send: stats +2025-01-06 20:08:27,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:08:27,702 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:08:27,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:28,736 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:28,775 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:29,736 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:30,736 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:31,736 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:32,736 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:33,737 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:34,015 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:34,350 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:34,363 DEBUG SenderThread:2104256 [sender.py:send():379] send: files +2025-01-06 20:08:34,363 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_e403f7a04c6fe992037c.mp4 with policy now +2025-01-06 20:08:34,367 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:08:34,368 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:08:34,368 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:08:34,375 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:08:34,648 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/gq1tw6b4-media/videos/eval/2_eval_2_e403f7a04c6fe992037c.mp4 +2025-01-06 20:08:34,740 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:35,350 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:08:35,350 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/2_eval_2_e403f7a04c6fe992037c.mp4 +2025-01-06 20:08:35,351 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval +2025-01-06 20:08:35,671 INFO wandb-upload_2:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/11wkk99u-media/videos/eval/2_eval_2_e403f7a04c6fe992037c.mp4 +2025-01-06 20:08:35,751 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:36,350 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:36,740 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:37,745 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:38,740 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:39,165 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:39,746 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:40,352 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:40,740 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:41,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:42,353 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:42,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:08:42,703 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:08:42,745 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:43,751 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:43,954 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:08:43,954 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:08:43,955 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:08:44,353 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:08:44,354 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:08:44,607 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/mcxyotei-events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:08:44,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:44,916 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:45,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:46,354 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:46,750 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:47,753 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:48,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:49,746 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:49,985 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:50,355 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:50,745 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:51,745 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:52,746 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:53,745 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:54,746 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:55,709 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:55,746 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:56,746 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:57,365 DEBUG SenderThread:2104256 [sender.py:send():379] send: stats +2025-01-06 20:08:57,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:08:57,702 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:08:57,746 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:58,358 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:58,747 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:59,747 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:00,747 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:00,849 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:01,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:02,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:03,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:04,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:05,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:06,710 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:06,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:07,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:08,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:09,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:10,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:11,710 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:11,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:12,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:09:12,703 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:09:12,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:13,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:14,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:15,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:16,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:16,796 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:17,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:18,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:19,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:20,750 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:21,750 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:21,975 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:22,366 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:09:22,750 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:23,749 DEBUG SenderThread:2104256 [sender.py:send():379] send: files +2025-01-06 20:09:23,749 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_fd7b6d2ce3d8ed8b882d.mp4 with policy now +2025-01-06 20:09:23,753 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:09:23,754 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:09:23,754 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:09:23,756 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:23,759 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:09:24,019 INFO wandb-upload_2:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/gw1d39bu-media/videos/eval/3_eval_3_fd7b6d2ce3d8ed8b882d.mp4 +2025-01-06 20:09:24,367 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:09:24,367 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:09:24,367 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/3_eval_3_fd7b6d2ce3d8ed8b882d.mp4 +2025-01-06 20:09:24,368 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval +2025-01-06 20:09:24,661 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/s1agq39r-media/videos/eval/3_eval_3_fd7b6d2ce3d8ed8b882d.mp4 +2025-01-06 20:09:24,754 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:25,759 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:26,368 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:09:26,760 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:27,365 DEBUG SenderThread:2104256 [sender.py:send():379] send: stats +2025-01-06 20:09:27,366 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:27,710 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:09:27,710 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:09:27,776 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:28,776 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:29,774 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:30,369 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:09:30,762 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:31,766 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:32,011 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:09:32,011 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:09:32,012 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:09:32,370 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:09:32,370 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:09:32,370 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:09:32,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:32,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:33,786 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:34,370 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:09:34,779 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:35,776 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:36,774 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:37,774 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:38,372 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:09:38,623 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:38,775 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:39,775 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:40,373 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:09:40,775 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:41,775 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:42,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:09:42,705 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:09:42,775 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:43,776 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:43,818 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:44,776 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:45,776 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:46,778 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:47,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:48,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:49,717 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:49,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:50,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:51,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:52,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:53,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:54,717 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:54,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:55,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:56,778 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:57,366 DEBUG SenderThread:2104256 [sender.py:send():379] send: stats +2025-01-06 20:09:57,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:09:57,705 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:09:57,778 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:58,778 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:59,771 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:59,778 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:00,778 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:01,778 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:02,778 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:03,779 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:04,779 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:05,718 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:10:05,779 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:06,779 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:07,779 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:08,780 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:09,779 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:10,784 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:11,488 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:10:11,784 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:12,225 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:10:12,277 DEBUG SenderThread:2104256 [sender.py:send():379] send: files +2025-01-06 20:10:12,306 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_7504f2a27cb28e455eb2.mp4 with policy now +2025-01-06 20:10:12,306 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:10:12,306 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:12,325 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:12,384 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:10:12,384 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:10:12,384 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/4_eval_4_7504f2a27cb28e455eb2.mp4 +2025-01-06 20:10:12,384 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval +2025-01-06 20:10:12,591 INFO wandb-upload_2:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/iqviflol-media/videos/eval/4_eval_4_7504f2a27cb28e455eb2.mp4 +2025-01-06 20:10:12,713 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:10:12,713 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:10:12,783 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:12,865 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/638dbn2c-media/videos/eval/4_eval_4_7504f2a27cb28e455eb2.mp4 +2025-01-06 20:10:13,799 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:14,385 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:10:14,787 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:15,785 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:16,791 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:17,161 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:10:17,809 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:18,386 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:10:18,793 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:19,798 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:20,387 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:10:20,806 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:21,809 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:22,072 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:10:22,073 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:22,073 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:22,219 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:10:22,219 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:10:22,375 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:10:22,388 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:10:22,388 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:10:22,388 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:10:22,388 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media +2025-01-06 20:10:22,388 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table +2025-01-06 20:10:22,388 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table/eval +2025-01-06 20:10:22,613 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/b1c4gnye-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:10:22,632 INFO wandb-upload_2:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/bpzc5pmd-events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:10:22,768 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:10:22,778 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:10:22,795 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:22,828 INFO wandb-upload_1:2104256 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_9qfkvb_ +2025-01-06 20:10:23,264 INFO SenderThread:2104256 [sender.py:send_request_log_artifact():1525] logged artifact run-m386gyxa-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjIyMjYxOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYwMDI0Mg==', 'latestArtifact': None}} +2025-01-06 20:10:23,264 DEBUG SenderThread:2104256 [sender.py:send():379] send: files +2025-01-06 20:10:23,264 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 20:10:23,265 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:10:23,265 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:10:23,265 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,265 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,266 DEBUG SenderThread:2104256 [sender.py:send():379] send: telemetry +2025-01-06 20:10:23,266 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,269 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,270 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,270 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,270 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,270 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,271 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,271 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,271 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,271 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,272 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,272 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,272 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,272 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,273 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,273 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,273 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,273 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,274 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,274 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,274 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,274 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,274 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:10:23,388 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:10:23,388 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:10:23,388 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table/eval/total_success_rate_ci_6_6e461524546d8e890233.table.json +2025-01-06 20:10:23,389 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table/eval +2025-01-06 20:10:23,513 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/4jyhaquy-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:10:23,581 INFO wandb-upload_1:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/nwy4c06a-media/table/eval/total_success_rate_ci_6_6e461524546d8e890233.table.json +2025-01-06 20:10:23,632 INFO wandb-upload_2:2104256 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmptzqsyq0c +2025-01-06 20:10:23,795 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:24,000 INFO SenderThread:2104256 [sender.py:send_request_log_artifact():1525] logged artifact run-m386gyxa-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjIyMjcwMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYwMDMwNw==', 'latestArtifact': None}} +2025-01-06 20:10:24,000 DEBUG SenderThread:2104256 [sender.py:send():379] send: files +2025-01-06 20:10:24,000 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_6e461524546d8e890233.table.json with policy now +2025-01-06 20:10:24,000 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:10:24,001 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:24,002 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:24,247 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/0tu7i3cn-media/table/eval/total_success_rate_ci_6_6e461524546d8e890233.table.json +2025-01-06 20:10:24,389 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:10:24,389 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:10:24,795 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:25,795 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:26,795 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:27,366 DEBUG SenderThread:2104256 [sender.py:send():379] send: stats +2025-01-06 20:10:27,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:10:27,706 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:10:27,796 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:28,178 DEBUG SenderThread:2104256 [sender.py:send():379] send: exit +2025-01-06 20:10:28,178 INFO SenderThread:2104256 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 20:10:28,178 INFO SenderThread:2104256 [sender.py:send_exit():588] handling runtime: 270 +2025-01-06 20:10:28,179 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:28,179 INFO SenderThread:2104256 [sender.py:send_exit():594] send defer +2025-01-06 20:10:28,179 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:28,179 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 20:10:28,180 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:28,180 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 20:10:28,180 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 1 +2025-01-06 20:10:28,180 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:28,180 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 20:10:28,180 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:28,180 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 20:10:28,180 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 2 +2025-01-06 20:10:28,180 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:28,180 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 20:10:28,180 INFO HandlerThread:2104256 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 20:10:28,180 DEBUG SystemMonitor:2104256 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 20:10:28,180 DEBUG SystemMonitor:2104256 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 20:10:28,181 INFO HandlerThread:2104256 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 20:10:28,181 INFO HandlerThread:2104256 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 20:10:28,233 INFO HandlerThread:2104256 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 20:10:28,233 INFO HandlerThread:2104256 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 20:10:28,233 INFO HandlerThread:2104256 [interfaces.py:finish():200] Joined network monitor +2025-01-06 20:10:28,234 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:28,234 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 20:10:28,234 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 3 +2025-01-06 20:10:28,234 DEBUG SenderThread:2104256 [sender.py:send():379] send: stats +2025-01-06 20:10:28,234 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:28,234 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 20:10:28,234 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:28,234 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 20:10:28,235 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 4 +2025-01-06 20:10:28,235 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:28,235 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 20:10:28,390 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:10:30,391 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:10:35,080 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:10:35,080 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:10:35,080 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 20:10:35,080 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:10:35,081 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:35,081 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 20:10:35,082 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 5 +2025-01-06 20:10:35,082 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:10:35,082 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:10:35,082 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:35,082 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:35,082 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 20:10:35,083 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:35,083 DEBUG SenderThread:2104256 [sender.py:send():379] send: summary +2025-01-06 20:10:35,083 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:35,084 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:35,084 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 20:10:35,084 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 6 +2025-01-06 20:10:35,084 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:35,084 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 20:10:35,084 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:35,084 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 20:10:35,087 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:10:35,189 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:10:35,204 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 7 +2025-01-06 20:10:35,204 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:10:35,204 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:35,204 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 20:10:35,205 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:35,205 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 20:10:35,393 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/config.yaml +2025-01-06 20:10:35,393 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:10:36,189 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:10:37,993 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 8 +2025-01-06 20:10:37,993 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:10:37,994 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:37,994 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 20:10:37,994 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:37,994 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 20:10:37,994 INFO SenderThread:2104256 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 20:10:37,997 INFO SenderThread:2104256 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 20:10:37,999 INFO SenderThread:2104256 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 20:10:38,008 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 9 +2025-01-06 20:10:38,008 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:38,008 DEBUG SenderThread:2104256 [sender.py:send():379] send: artifact +2025-01-06 20:10:38,008 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 20:10:38,190 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:10:38,224 INFO SenderThread:2104256 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 20:10:38,224 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:38,224 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 20:10:38,224 INFO SenderThread:2104256 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 20:10:38,394 INFO SenderThread:2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:10:38,394 INFO SenderThread:2104256 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files +2025-01-06 20:10:38,395 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/config.yaml config.yaml +2025-01-06 20:10:38,395 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json wandb-summary.json +2025-01-06 20:10:38,395 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/requirements.txt requirements.txt +2025-01-06 20:10:38,395 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-metadata.json wandb-metadata.json +2025-01-06 20:10:38,395 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:10:38,395 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log output.log +2025-01-06 20:10:38,396 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/diff.patch diff.patch +2025-01-06 20:10:38,402 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 20:10:38,402 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/3_eval_3_fd7b6d2ce3d8ed8b882d.mp4 media/videos/eval/3_eval_3_fd7b6d2ce3d8ed8b882d.mp4 +2025-01-06 20:10:38,402 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/4_eval_4_7504f2a27cb28e455eb2.mp4 media/videos/eval/4_eval_4_7504f2a27cb28e455eb2.mp4 +2025-01-06 20:10:38,402 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/0_eval_0_22bed2761bf0b211c58d.mp4 media/videos/eval/0_eval_0_22bed2761bf0b211c58d.mp4 +2025-01-06 20:10:38,402 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/1_eval_1_ef07a287a59b234dfd50.mp4 media/videos/eval/1_eval_1_ef07a287a59b234dfd50.mp4 +2025-01-06 20:10:38,403 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/2_eval_2_e403f7a04c6fe992037c.mp4 media/videos/eval/2_eval_2_e403f7a04c6fe992037c.mp4 +2025-01-06 20:10:38,403 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table/eval/total_success_rate_ci_6_6e461524546d8e890233.table.json media/table/eval/total_success_rate_ci_6_6e461524546d8e890233.table.json +2025-01-06 20:10:38,403 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:10:38,403 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 10 +2025-01-06 20:10:38,403 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:10:38,403 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:38,403 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 20:10:38,405 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:38,405 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 20:10:38,405 INFO SenderThread:2104256 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:10:38,592 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/requirements.txt +2025-01-06 20:10:38,599 INFO wandb-upload_2:2104256 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/config.yaml +2025-01-06 20:10:38,617 INFO wandb-upload_4:2104256 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:10:38,638 INFO wandb-upload_1:2104256 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:10:38,684 INFO wandb-upload_3:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/mksmfkw2-events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:10:38,885 INFO Thread-11 (_thread_body):2104256 [sender.py:transition_state():614] send defer: 11 +2025-01-06 20:10:38,885 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:38,885 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 20:10:38,885 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:38,885 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 20:10:38,885 INFO SenderThread:2104256 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 20:10:38,886 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 12 +2025-01-06 20:10:38,886 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:38,887 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 20:10:38,887 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:38,887 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 20:10:38,887 INFO SenderThread:2104256 [file_stream.py:finish():601] file stream finish called +2025-01-06 20:10:38,942 INFO SenderThread:2104256 [file_stream.py:finish():605] file stream finish is done +2025-01-06 20:10:38,943 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 13 +2025-01-06 20:10:38,943 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:38,943 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 20:10:38,943 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:38,943 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 20:10:38,943 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 14 +2025-01-06 20:10:38,943 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:38,943 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 20:10:38,944 DEBUG SenderThread:2104256 [sender.py:send():379] send: final +2025-01-06 20:10:38,944 DEBUG SenderThread:2104256 [sender.py:send():379] send: footer +2025-01-06 20:10:38,944 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:38,944 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 20:10:38,944 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:10:38,944 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:10:38,944 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 20:10:38,945 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: server_info +2025-01-06 20:10:38,945 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:10:38,946 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 20:10:38,947 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 20:10:38,947 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:38,992 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:10:38,993 INFO MainThread:2104256 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 20:10:38,993 INFO MainThread:2104256 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 20:10:38,993 INFO MainThread:2104256 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 20:10:38,994 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 20:10:38,994 INFO HandlerThread:2104256 [handler.py:finish():882] shutting down handler +2025-01-06 20:10:39,947 INFO WriterThread:2104256 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/run-m386gyxa.wandb +2025-01-06 20:10:39,993 INFO SenderThread:2104256 [sender.py:finish():1615] shutting down sender +2025-01-06 20:10:39,993 INFO SenderThread:2104256 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:10:39,993 INFO SenderThread:2104256 [file_pusher.py:join():175] waiting for file pusher diff --git a/m386gyxa/wandb/debug.log b/m386gyxa/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..be44991f25d7fcd08eee1a76f1eea6bd52734196 --- /dev/null +++ b/m386gyxa/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 20:05:57,038 INFO MainThread:2104083 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 20:05:57,038 INFO MainThread:2104083 [wandb_setup.py:_flush():76] Configure stats pid to 2104083 +2025-01-06 20:05:57,038 INFO MainThread:2104083 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 20:05:57,039 INFO MainThread:2104083 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/settings +2025-01-06 20:05:57,039 INFO MainThread:2104083 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 20:05:57,039 INFO MainThread:2104083 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 20:05:57,039 INFO MainThread:2104083 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 20:05:57,039 INFO MainThread:2104083 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 20:05:57,039 INFO MainThread:2104083 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/logs/debug.log +2025-01-06 20:05:57,039 INFO MainThread:2104083 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/logs/debug-internal.log +2025-01-06 20:05:57,039 INFO MainThread:2104083 [wandb_init.py:init():569] calling init triggers +2025-01-06 20:05:57,040 INFO MainThread:2104083 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa'} +2025-01-06 20:05:57,040 INFO MainThread:2104083 [wandb_init.py:init():619] starting backend +2025-01-06 20:05:57,040 INFO MainThread:2104083 [wandb_init.py:init():623] setting up manager +2025-01-06 20:05:57,044 INFO MainThread:2104083 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 20:05:57,045 INFO MainThread:2104083 [wandb_init.py:init():631] backend started and connected +2025-01-06 20:05:57,059 INFO MainThread:2104083 [wandb_init.py:init():720] updated telemetry +2025-01-06 20:05:57,067 INFO MainThread:2104083 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 20:05:57,308 INFO MainThread:2104083 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 20:05:57,347 INFO MainThread:2104083 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 20:05:57,348 INFO MainThread:2104083 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 20:05:57,700 INFO MainThread:2104083 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 20:05:57,701 INFO MainThread:2104083 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 20:05:57,701 INFO MainThread:2104083 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 20:05:57,701 INFO MainThread:2104083 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 20:05:57,702 INFO MainThread:2104083 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 20:05:57,702 INFO MainThread:2104083 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/tb, True +2025-01-06 20:06:02,408 INFO MainThread:2104083 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-06 20:10:39,995 WARNING MsgRouterThr:2104083 [router.py:message_loop():77] message_loop has been closed diff --git a/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/code/FISH/eval_robot.py b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/config.yaml b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd5a95c8a16b80ff6ba5fc5469568771dc942969 --- /dev/null +++ b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1017_0' +wandb_notes: + desc: null + value: 1017_0_restarted_11 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736211957 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/diff.patch b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..4c1a5f92eb09cddbc2402d3534362390d486902d --- /dev/null +++ b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..54e6cd0 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table/eval/total_success_rate_ci_6_6e461524546d8e890233.table.json b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table/eval/total_success_rate_ci_6_6e461524546d8e890233.table.json new file mode 100644 index 0000000000000000000000000000000000000000..24f4f56821632c2ce011c8cdf9e44aab6a073a01 --- /dev/null +++ b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table/eval/total_success_rate_ci_6_6e461524546d8e890233.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[20, 20, 1.0, 0.8122211992116188, 1, 0.8828159271905114, 1, 0.9095175488964253, 1, 0.9018393158108982, 1]]} \ No newline at end of file diff --git a/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/0_eval_0_22bed2761bf0b211c58d.mp4 b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/0_eval_0_22bed2761bf0b211c58d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0d2ddc438a72922b2b81ebdb56d79a7cf651562d --- /dev/null +++ b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/0_eval_0_22bed2761bf0b211c58d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22bed2761bf0b211c58d04b8679118dbf8826eb7892546c26f648b5b459f41c8 +size 1459978 diff --git a/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/1_eval_1_ef07a287a59b234dfd50.mp4 b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/1_eval_1_ef07a287a59b234dfd50.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7fd46c2cd325290dfd47f5eea99d6b64a31d17e1 --- /dev/null +++ b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/1_eval_1_ef07a287a59b234dfd50.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef07a287a59b234dfd5075a2a428c07672ce90dc268a4c8ca8b99b816cd5387c +size 1455390 diff --git a/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/2_eval_2_e403f7a04c6fe992037c.mp4 b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/2_eval_2_e403f7a04c6fe992037c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..17944b2cffd0612103dbef1b60df3367b5043a0d --- /dev/null +++ b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/2_eval_2_e403f7a04c6fe992037c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e403f7a04c6fe992037c52f56c3716cf384e15d5c22c3529b37c30eab8242766 +size 1450880 diff --git a/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/3_eval_3_fd7b6d2ce3d8ed8b882d.mp4 b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/3_eval_3_fd7b6d2ce3d8ed8b882d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a7ab7d1bcf3b66a3b0a6c2a5690dcb62b9c0ef31 --- /dev/null +++ b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/3_eval_3_fd7b6d2ce3d8ed8b882d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd7b6d2ce3d8ed8b882db063488e1aae0a7bc7de648f5503db03033e05b6e9ac +size 1462155 diff --git a/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/4_eval_4_7504f2a27cb28e455eb2.mp4 b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/4_eval_4_7504f2a27cb28e455eb2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df0de8ec97964bc98f8320f5fb5f6931cb130eeb --- /dev/null +++ b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/4_eval_4_7504f2a27cb28e455eb2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7504f2a27cb28e455eb230f84c964216f73462455eb50dd74e4d6071cc9566c8 +size 1474668 diff --git a/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..74d2974c4167d16f28576b9841cd51e6d7dcf55b --- /dev/null +++ b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log @@ -0,0 +1,286 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736211962.409128]: resetting environment +[INFO] [1736211962.413750]: cleared current plan +[INFO] [1736211962.415800]: moving to home +[INFO] [1736211963.418574]: reached home +[INFO] [1736211963.420578]: reset action history +[INFO] [1736211966.131255]: environment reset +Starting episode 0 +[INFO] [1736211966.135452]: resetting environment +[INFO] [1736211966.139475]: cleared current plan +[INFO] [1736211966.140381]: moving to home +[INFO] [1736211967.141657]: reached home +[INFO] [1736211967.146065]: reset action history +[INFO] [1736211969.857033]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736211977.720926]: resetting environment +[INFO] [1736211977.724900]: cleared current plan +[INFO] [1736211977.726071]: moving to home +[INFO] [1736211978.728034]: reached home +[INFO] [1736211978.729478]: reset action history +[INFO] [1736211981.488626]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/episode_rosbags/episode_0_2025-01-06-20-06-21.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2104083.log +started roslaunch server http://158.130.50.37:40909/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2104386] +process[print_text-2]: started with pid [2104387] +process[rosbag_record-3]: started with pid [2104388] +started bagging! +For topic gripper_width: timestamp difference is 100406569 for nearest: 1736212003399948758 - target: 1736212003299542189 at idx 249 +For topic gripper_width: timestamp difference is 101427751 for nearest: 1736212004699917989 - target: 1736212004598490238 at idx 249 +For topic gripper_width: timestamp difference is 102427830 for nearest: 1736212005999878516 - target: 1736212005897450686 at idx 249 +[INFO] [1736212016.375529]: Storing episode... +[pub_text-1] killing on exit +[print_text-2] killing on exit +[rosbag_record-3] killing on exit +[INFO] [1736212017.170398]: Stored episode 1. +[INFO] [1736212017.170670]: Saving video... +[INFO] [1736212017.358714]: Video saved! +Starting episode 1 +[INFO] [1736212017.373014]: resetting environment +[INFO] [1736212017.387698]: cleared current plan +[INFO] [1736212017.390030]: moving to home +[INFO] [1736212020.992511]: reached home +[INFO] [1736212020.992975]: reset action history +[INFO] [1736212023.703362]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736212025.820358]: resetting environment +[INFO] [1736212025.824301]: cleared current plan +[INFO] [1736212025.824476]: moving to home +[INFO] [1736212026.825269]: reached home +[INFO] [1736212026.825678]: reset action history +[INFO] [1736212029.535345]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/episode_rosbags/episode_1_2025-01-06-20-07-09.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2104083.log +started roslaunch server http://158.130.50.37:40295/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2104511] +process[print_text-5]: started with pid [2104513] +process[rosbag_record-6]: started with pid [2104537] +started bagging! +For topic gripper_width: timestamp difference is 103045272 for nearest: 1736212048033249187 - target: 1736212047930203915 at idx 243 +For topic gripper_width: timestamp difference is 104326035 for nearest: 1736212049599950100 - target: 1736212049495624065 at idx 249 +[INFO] [1736212064.504725]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736212065.292146]: Stored episode 2. +[INFO] [1736212065.295267]: Saving video... +[WARN] [1736212065.828195]: Plan exhausted +[WARN] [1736212065.868086]: Plan exhausted +[WARN] [1736212065.907559]: Plan exhausted +[WARN] [1736212065.947544]: Plan exhausted +[WARN] [1736212065.988472]: Plan exhausted +[INFO] [1736212065.982901]: Video saved! +Starting episode 2 +[INFO] [1736212066.008194]: resetting environment +[INFO] [1736212066.019391]: cleared current plan +[INFO] [1736212066.019635]: moving to home +[INFO] [1736212069.727933]: reached home +[INFO] [1736212069.729000]: reset action history +[INFO] [1736212072.523851]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736212074.678904]: resetting environment +[INFO] [1736212074.698809]: cleared current plan +[INFO] [1736212074.700431]: moving to home +[INFO] [1736212075.705269]: reached home +[INFO] [1736212075.705708]: reset action history +[INFO] [1736212078.418114]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/episode_rosbags/episode_2_2025-01-06-20-07-58.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2104083.log +started roslaunch server http://158.130.50.37:43479/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2104655] +process[print_text-8]: started with pid [2104656] +process[rosbag_record-9]: started with pid [2104680] +started bagging! +For topic gripper_width: timestamp difference is 101144259 for nearest: 1736212086899875109 - target: 1736212086798730850 at idx 113 +For topic gripper_width: timestamp difference is 102642553 for nearest: 1736212088733250396 - target: 1736212088630607843 at idx 133 +For topic gripper_width: timestamp difference is 103670253 for nearest: 1736212090033207072 - target: 1736212089929536819 at idx 151 +For topic gripper_width: timestamp difference is 106053080 for nearest: 1736212092899917092 - target: 1736212092793864012 at idx 192 +[INFO] [1736212113.213926]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736212114.015371]: Stored episode 3. +[INFO] [1736212114.017801]: Saving video... +[INFO] [1736212114.285974]: Video saved! +Starting episode 3 +[INFO] [1736212114.379825]: resetting environment +[INFO] [1736212114.414016]: cleared current plan +[INFO] [1736212114.415012]: moving to home +[INFO] [1736212118.124776]: reached home +[INFO] [1736212118.130305]: reset action history +[INFO] [1736212120.921705]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736212123.904800]: resetting environment +[INFO] [1736212123.913224]: cleared current plan +[INFO] [1736212123.913428]: moving to home +[INFO] [1736212124.915921]: reached home +[INFO] [1736212124.918407]: reset action history +[INFO] [1736212127.707893]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/episode_rosbags/episode_3_2025-01-06-20-08-47.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2104083.log +started roslaunch server http://158.130.50.37:43211/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2104797] +process[print_text-11]: started with pid [2104798] +process[rosbag_record-12]: started with pid [2104822] +started bagging! +For topic gripper_width: timestamp difference is -100520389 for nearest: 1736212136833279593 - target: 1736212136933799982 at idx 124 +For topic gripper_width: timestamp difference is -105649920 for nearest: 1736212160733250408 - target: 1736212160838900328 at idx 249 +[INFO] [1736212162.463824]: Storing episode... +[pub_text-10] killing on exit +[print_text-11] killing on exit[rosbag_record-12] killing on exit +[INFO] [1736212163.317991]: Stored episode 4. +[INFO] [1736212163.326046]: Saving video... +[INFO] [1736212163.689291]: Video saved! +Starting episode 4 +[INFO] [1736212163.755212]: resetting environment +[INFO] [1736212163.800597]: cleared current plan +[INFO] [1736212163.812663]: moving to home +[INFO] [1736212167.521073]: reached home +[INFO] [1736212167.525148]: reset action history +[INFO] [1736212170.374985]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736212171.734649]: resetting environment +[INFO] [1736212171.785360]: cleared current plan +[INFO] [1736212171.791449]: moving to home +[INFO] [1736212172.799794]: reached home +[INFO] [1736212172.807673]: reset action history +[INFO] [1736212175.595163]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/episode_rosbags/episode_4_2025-01-06-20-09-35.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2104083.log +started roslaunch server http://158.130.50.37:45515/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2104961] +process[print_text-14]: started with pid [2104962] +process[rosbag_record-15]: started with pid [2104963] +started bagging! +For topic gripper_width: timestamp difference is -100826752 for nearest: 1736212178433278072 - target: 1736212178534104824 at idx 34 +[INFO] [1736212210.614638]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1736212211.487715]: Stored episode 5. +[INFO] [1736212211.499201]: Saving video... +[WARN] [1736212211.947296]: Plan exhausted +[WARN] [1736212211.988897]: Plan exhausted +[WARN] [1736212212.027294]: Plan exhausted +[WARN] [1736212212.077418]: Plan exhausted +[WARN] [1736212212.108680]: Plan exhausted +[WARN] [1736212212.150344]: Plan exhausted +[WARN] [1736212212.189750]: Plan exhausted +[WARN] [1736212212.232782]: Plan exhausted +[WARN] [1736212212.267373]: Plan exhausted +[INFO] [1736212212.165231]: Video saved! +[INFO] [1736212212.227427]: resetting environment +[INFO] [1736212212.274187]: cleared current plan +[INFO] [1736212212.294114]: moving to home +[INFO] [1736212216.112451]: reached home +[INFO] [1736212216.119796]: reset action history +[INFO] [1736212218.969682]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/requirements.txt b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-metadata.json b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..11849af78a8105b51b86b82fe7b9cd443cf9272e --- /dev/null +++ b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T01:05:57.372330", + "startedAt": "2025-01-07T01:05:57.037280", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.1145, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.859, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 438.2542266845703 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4ee700385d8d7845ab5cacc66e63ff621195887f --- /dev/null +++ b/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "22bed2761bf0b211c58d04b8679118dbf8826eb7892546c26f648b5b459f41c8", "size": 1459978, "path": "media/videos/eval/0_eval_0_22bed2761bf0b211c58d.mp4"}, "global_step": 665, "_timestamp": 1736212222.6911218, "_runtime": 265.6456549167633, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "ef07a287a59b234dfd5075a2a428c07672ce90dc268a4c8ca8b99b816cd5387c", "size": 1455390, "path": "media/videos/eval/1_eval_1_ef07a287a59b234dfd50.mp4"}, "eval/num_success": 5.0, "episode": 20.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "e403f7a04c6fe992037c52f56c3716cf384e15d5c22c3529b37c30eab8242766", "size": 1450880, "path": "media/videos/eval/2_eval_2_e403f7a04c6fe992037c.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "fd7b6d2ce3d8ed8b882db063488e1aae0a7bc7de648f5503db03033e05b6e9ac", "size": 1462155, "path": "media/videos/eval/3_eval_3_fd7b6d2ce3d8ed8b882d.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "7504f2a27cb28e455eb230f84c964216f73462455eb50dd74e4d6071cc9566c8", "size": 1474668, "path": "media/videos/eval/4_eval_4_7504f2a27cb28e455eb2.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://ewapp74aym759lkxueavtdx700kdqnem05jgpscwchy6y7ybqjdmfhfu2zyy5gj9x16qtrsx0zttfz3f02ao4babvnaph7scoirsy6w5dmlvt4wcnjlv2r1cru45m5ji/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://wwodp1uc9r8kefuwrxrysaufk9qjs301ixnnw5qnv60s6sybjlv757vq80znbm1x65qbk9engk85vgedaks1awvx183beut3d48zqq31udt212fgiqoor9z22ljkb1nq:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 20, "total_num_episodes": 20, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.8122211992116188, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.8828159271905114, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.9095175488964253, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 0.9018393158108982, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "6e461524546d8e890233269491df5e4f676241d4ded777aff83153f1249061e1", "size": 437, "artifact_path": "wandb-client-artifact://4pwtang0ymrve4b161djwbtovvazajpfpwxw3zy4cpoaiwxke11gdovtqg07h3qfgfp99r84o9iqd6x6jd0ei5mnjp4t7lraogpp3de99xsb2okqo7uhfo66alupks9q/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://aauysz4h9v26qx6p76bvakld7e87t9i0u2lnir97u49ty3cluj0ej9pperiyyxrsw54jgu2v8dlbng6d5gl1y8pfxizeki4ryoh5jys70siylne6gmxvpa1nx71o6w9o:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_6e461524546d8e890233.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 270}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.8122211694717407, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.8828158974647522, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.9095175266265869, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 0.9018393158912659, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/m386gyxa/wandb/run-20250106_200557-m386gyxa/logs/debug-internal.log b/m386gyxa/wandb/run-20250106_200557-m386gyxa/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4f079ea180eb3d516dbd6e5c9d542e900e8eb337 --- /dev/null +++ b/m386gyxa/wandb/run-20250106_200557-m386gyxa/logs/debug-internal.log @@ -0,0 +1,781 @@ +2025-01-06 20:05:57,045 INFO StreamThr :2104256 [internal.py:wandb_internal():85] W&B internal server running at pid: 2104256, started at: 2025-01-06 20:05:57.044943 +2025-01-06 20:05:57,046 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status +2025-01-06 20:05:57,060 INFO WriterThread:2104256 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/run-m386gyxa.wandb +2025-01-06 20:05:57,060 DEBUG SenderThread:2104256 [sender.py:send():379] send: header +2025-01-06 20:05:57,069 DEBUG SenderThread:2104256 [sender.py:send():379] send: run +2025-01-06 20:05:57,294 INFO SenderThread:2104256 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files +2025-01-06 20:05:57,295 INFO SenderThread:2104256 [sender.py:_start_run_threads():1188] run started: m386gyxa with start time 1736211957.045467 +2025-01-06 20:05:57,309 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 20:05:57,309 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: check_version +2025-01-06 20:05:57,354 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 20:05:57,359 DEBUG HandlerThread:2104256 [system_info.py:__init__():26] System info init +2025-01-06 20:05:57,359 DEBUG HandlerThread:2104256 [system_info.py:__init__():41] System info init done +2025-01-06 20:05:57,359 INFO HandlerThread:2104256 [system_monitor.py:start():194] Starting system monitor +2025-01-06 20:05:57,360 INFO SystemMonitor:2104256 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 20:05:57,360 INFO HandlerThread:2104256 [system_monitor.py:probe():214] Collecting system info +2025-01-06 20:05:57,360 INFO SystemMonitor:2104256 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 20:05:57,361 INFO SystemMonitor:2104256 [interfaces.py:start():188] Started disk monitoring +2025-01-06 20:05:57,361 INFO SystemMonitor:2104256 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 20:05:57,362 INFO SystemMonitor:2104256 [interfaces.py:start():188] Started memory monitoring +2025-01-06 20:05:57,362 INFO SystemMonitor:2104256 [interfaces.py:start():188] Started network monitoring +2025-01-06 20:05:57,372 DEBUG HandlerThread:2104256 [system_info.py:probe():152] Probing system +2025-01-06 20:05:57,373 DEBUG HandlerThread:2104256 [system_info.py:_probe_git():137] Probing git +2025-01-06 20:05:57,378 DEBUG HandlerThread:2104256 [system_info.py:_probe_git():145] Probing git done +2025-01-06 20:05:57,378 DEBUG HandlerThread:2104256 [system_info.py:probe():200] Probing system done +2025-01-06 20:05:57,378 DEBUG HandlerThread:2104256 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T01:05:57.372330', 'startedAt': '2025-01-07T01:05:57.037280', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.1145, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.859, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 438.2542266845703}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 20:05:57,378 INFO HandlerThread:2104256 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 20:05:57,378 INFO HandlerThread:2104256 [system_monitor.py:probe():227] Publishing system info +2025-01-06 20:05:57,378 DEBUG HandlerThread:2104256 [system_info.py:_save_code():44] Saving code +2025-01-06 20:05:57,379 DEBUG HandlerThread:2104256 [system_info.py:_save_code():67] Saving code done +2025-01-06 20:05:57,380 DEBUG HandlerThread:2104256 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 20:05:57,447 DEBUG HandlerThread:2104256 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 20:05:57,448 INFO HandlerThread:2104256 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 20:05:57,451 DEBUG SenderThread:2104256 [sender.py:send():379] send: files +2025-01-06 20:05:57,451 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 20:05:57,451 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 20:05:57,452 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 20:05:57,700 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 20:05:57,700 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: python_packages +2025-01-06 20:05:57,700 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:05:57,701 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:05:57,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:05:57,703 INFO HandlerThread:2104256 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/tb" + save: true +} +control { + relay_id: "e67a100e33da4f33931342645f4e8e07" +} +_info { + stream_id: "m386gyxa" +} + +2025-01-06 20:05:57,703 DEBUG HandlerThread:2104256 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 20:05:57,712 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/u85604lb-wandb-metadata.json +2025-01-06 20:05:57,732 INFO wandb-upload_1:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/uv7lvji5-code/FISH/eval_robot.py +2025-01-06 20:05:57,753 INFO wandb-upload_2:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/jhao2ns6-diff.patch +2025-01-06 20:05:57,766 DEBUG SenderThread:2104256 [sender.py:send():379] send: telemetry +2025-01-06 20:05:57,766 DEBUG SenderThread:2104256 [sender.py:send():379] send: tbrecord +2025-01-06 20:05:57,767 DEBUG SenderThread:2104256 [sender.py:send():379] send: files +2025-01-06 20:05:57,767 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 with policy live +2025-01-06 20:05:57,987 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/mplqfvx3-events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:05:58,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/requirements.txt +2025-01-06 20:05:58,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/diff.patch +2025-01-06 20:05:58,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-metadata.json +2025-01-06 20:05:58,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/code/FISH/eval_robot.py +2025-01-06 20:05:58,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:05:58,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:05:58,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/code/FISH +2025-01-06 20:05:58,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/code +2025-01-06 20:05:58,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:05:59,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:00,296 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:00,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:01,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:02,408 DEBUG SenderThread:2104256 [sender.py:send():379] send: config +2025-01-06 20:06:02,409 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:02,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:03,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:04,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:04,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:05,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:06,297 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:06,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:07,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:07,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:08,298 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:08,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:09,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:10,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:11,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:12,299 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:12,700 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:06:12,700 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:06:12,741 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:12,799 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:13,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:14,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:15,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:16,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:17,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:18,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:18,728 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:19,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:20,301 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:20,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:21,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:22,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:23,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:24,302 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:24,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:24,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:25,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:26,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:27,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:06:27,701 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:06:27,741 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:28,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:29,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:29,826 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:30,304 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/config.yaml +2025-01-06 20:06:30,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:31,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:32,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:33,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:34,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:35,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:35,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:36,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:37,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:38,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:39,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:40,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:40,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:41,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:42,700 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:06:42,701 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:06:42,741 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:43,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:44,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:45,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:45,753 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:46,308 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:46,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:47,706 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:48,309 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:48,707 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:49,707 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:50,707 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:51,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:51,707 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:52,708 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:53,709 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:54,711 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:55,710 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:56,710 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:57,170 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:06:57,362 DEBUG SystemMonitor:2104256 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 20:06:57,363 DEBUG SenderThread:2104256 [sender.py:send():379] send: stats +2025-01-06 20:06:57,372 DEBUG SenderThread:2104256 [sender.py:send():379] send: files +2025-01-06 20:06:57,372 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_22bed2761bf0b211c58d.mp4 with policy now +2025-01-06 20:06:57,372 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:06:57,374 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:06:57,374 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:06:57,374 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:06:57,674 INFO wandb-upload_2:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/9zghjcpi-media/videos/eval/0_eval_0_22bed2761bf0b211c58d.mp4 +2025-01-06 20:06:57,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:06:57,702 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:06:57,710 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:58,311 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:06:58,312 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/0_eval_0_22bed2761bf0b211c58d.mp4 +2025-01-06 20:06:58,312 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:06:58,312 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media +2025-01-06 20:06:58,312 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval +2025-01-06 20:06:58,312 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos +2025-01-06 20:06:58,630 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/93kmdg7n-media/videos/eval/0_eval_0_22bed2761bf0b211c58d.mp4 +2025-01-06 20:06:58,710 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:06:59,710 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:00,312 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:00,710 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:01,711 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:02,313 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:02,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:02,711 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:03,745 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:04,711 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:05,711 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:06,314 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:06,314 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:07:06,571 INFO wandb-upload_2:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/k2gcvpd2-events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:07:06,711 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:07,711 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:08,315 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:08,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:08,712 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:09,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:10,717 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:11,717 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:12,316 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:12,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:07:12,702 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:07:12,745 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:13,717 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:13,758 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:14,718 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:15,718 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:16,718 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:17,718 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:18,718 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:19,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:19,718 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:20,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:21,719 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:22,719 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:23,719 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:24,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:24,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:25,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:26,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:27,364 DEBUG SenderThread:2104256 [sender.py:send():379] send: stats +2025-01-06 20:07:27,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:07:27,702 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:07:27,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:28,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:29,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:30,322 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:30,653 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:30,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:31,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:32,323 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:32,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:33,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:34,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:35,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:35,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:36,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:37,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:38,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:39,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:40,704 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:40,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:41,721 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:42,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:07:42,701 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:07:42,745 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:43,722 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:44,722 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:45,722 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:45,829 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:46,006 DEBUG SenderThread:2104256 [sender.py:send():379] send: files +2025-01-06 20:07:46,006 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_ef07a287a59b234dfd50.mp4 with policy now +2025-01-06 20:07:46,006 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:07:46,007 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:07:46,007 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:07:46,012 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:07:46,286 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/dwi7bziw-media/videos/eval/1_eval_1_ef07a287a59b234dfd50.mp4 +2025-01-06 20:07:46,333 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:07:46,334 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:46,334 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/1_eval_1_ef07a287a59b234dfd50.mp4 +2025-01-06 20:07:46,334 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval +2025-01-06 20:07:46,641 INFO wandb-upload_2:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/tmycjluj-media/videos/eval/1_eval_1_ef07a287a59b234dfd50.mp4 +2025-01-06 20:07:46,726 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:47,731 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:48,334 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:48,731 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:49,769 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:50,732 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:51,731 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:51,769 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:52,335 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:52,731 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:53,732 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:54,336 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:54,732 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:55,113 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:07:55,114 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:07:55,114 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:07:55,337 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:07:55,337 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:07:55,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:56,337 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:56,732 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:57,364 DEBUG SenderThread:2104256 [sender.py:send():379] send: stats +2025-01-06 20:07:57,365 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:07:57,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:07:57,702 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:07:57,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:58,338 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:07:58,732 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:07:59,732 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:00,338 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:00,732 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:01,732 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:02,675 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:02,733 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:03,733 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:04,733 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:05,733 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:06,733 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:07,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:07,733 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:08,341 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:08,733 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:09,734 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:10,342 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:10,734 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:11,734 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:12,342 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:12,701 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:08:12,702 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:08:12,745 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:12,776 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:13,734 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:14,343 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:14,734 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:15,734 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:16,734 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:17,734 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:18,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:18,735 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:19,735 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:20,735 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:21,735 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:22,735 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:23,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:23,735 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:24,735 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:25,735 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:26,736 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:27,365 DEBUG SenderThread:2104256 [sender.py:send():379] send: stats +2025-01-06 20:08:27,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:08:27,702 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:08:27,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:28,736 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:28,775 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:29,736 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:30,736 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:31,736 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:32,736 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:33,737 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:34,015 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:34,350 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:34,363 DEBUG SenderThread:2104256 [sender.py:send():379] send: files +2025-01-06 20:08:34,363 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_e403f7a04c6fe992037c.mp4 with policy now +2025-01-06 20:08:34,367 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:08:34,368 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:08:34,368 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:08:34,375 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:08:34,648 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/gq1tw6b4-media/videos/eval/2_eval_2_e403f7a04c6fe992037c.mp4 +2025-01-06 20:08:34,740 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:35,350 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:08:35,350 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/2_eval_2_e403f7a04c6fe992037c.mp4 +2025-01-06 20:08:35,351 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval +2025-01-06 20:08:35,671 INFO wandb-upload_2:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/11wkk99u-media/videos/eval/2_eval_2_e403f7a04c6fe992037c.mp4 +2025-01-06 20:08:35,751 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:36,350 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:36,740 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:37,745 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:38,740 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:39,165 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:39,746 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:40,352 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:40,740 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:41,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:42,353 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:42,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:08:42,703 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:08:42,745 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:43,751 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:43,954 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:08:43,954 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:08:43,955 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:08:44,353 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:08:44,354 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:08:44,607 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/mcxyotei-events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:08:44,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:44,916 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:45,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:46,354 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:46,750 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:47,753 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:48,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:49,746 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:49,985 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:50,355 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:50,745 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:51,745 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:52,746 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:53,745 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:54,746 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:55,709 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:08:55,746 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:56,746 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:57,365 DEBUG SenderThread:2104256 [sender.py:send():379] send: stats +2025-01-06 20:08:57,702 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:08:57,702 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:08:57,746 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:58,358 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:08:58,747 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:08:59,747 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:00,747 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:00,849 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:01,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:02,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:03,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:04,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:05,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:06,710 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:06,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:07,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:08,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:09,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:10,748 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:11,710 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:11,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:12,703 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:09:12,703 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:09:12,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:13,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:14,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:15,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:16,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:16,796 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:17,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:18,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:19,749 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:20,750 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:21,750 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:21,975 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:22,366 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:09:22,750 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:23,749 DEBUG SenderThread:2104256 [sender.py:send():379] send: files +2025-01-06 20:09:23,749 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_fd7b6d2ce3d8ed8b882d.mp4 with policy now +2025-01-06 20:09:23,753 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:09:23,754 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:09:23,754 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:09:23,756 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:23,759 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:09:24,019 INFO wandb-upload_2:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/gw1d39bu-media/videos/eval/3_eval_3_fd7b6d2ce3d8ed8b882d.mp4 +2025-01-06 20:09:24,367 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:09:24,367 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:09:24,367 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/3_eval_3_fd7b6d2ce3d8ed8b882d.mp4 +2025-01-06 20:09:24,368 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval +2025-01-06 20:09:24,661 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/s1agq39r-media/videos/eval/3_eval_3_fd7b6d2ce3d8ed8b882d.mp4 +2025-01-06 20:09:24,754 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:25,759 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:26,368 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:09:26,760 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:27,365 DEBUG SenderThread:2104256 [sender.py:send():379] send: stats +2025-01-06 20:09:27,366 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:27,710 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:09:27,710 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:09:27,776 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:28,776 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:29,774 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:30,369 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:09:30,762 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:31,766 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:32,011 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:09:32,011 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:09:32,012 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:09:32,370 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:09:32,370 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:09:32,370 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:09:32,720 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:32,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:33,786 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:34,370 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:09:34,779 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:35,776 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:36,774 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:37,774 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:38,372 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:09:38,623 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:38,775 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:39,775 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:40,373 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:09:40,775 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:41,775 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:42,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:09:42,705 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:09:42,775 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:43,776 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:43,818 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:44,776 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:45,776 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:46,778 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:47,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:48,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:49,717 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:49,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:50,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:51,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:52,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:53,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:54,717 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:54,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:55,777 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:56,778 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:57,366 DEBUG SenderThread:2104256 [sender.py:send():379] send: stats +2025-01-06 20:09:57,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:09:57,705 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:09:57,778 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:58,778 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:09:59,771 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:09:59,778 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:00,778 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:01,778 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:02,778 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:03,779 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:04,779 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:05,718 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:10:05,779 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:06,779 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:07,779 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:08,780 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:09,779 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:10,784 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:11,488 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:10:11,784 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:12,225 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:10:12,277 DEBUG SenderThread:2104256 [sender.py:send():379] send: files +2025-01-06 20:10:12,306 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_7504f2a27cb28e455eb2.mp4 with policy now +2025-01-06 20:10:12,306 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:10:12,306 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:12,325 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:12,384 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:10:12,384 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:10:12,384 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/4_eval_4_7504f2a27cb28e455eb2.mp4 +2025-01-06 20:10:12,384 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval +2025-01-06 20:10:12,591 INFO wandb-upload_2:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/iqviflol-media/videos/eval/4_eval_4_7504f2a27cb28e455eb2.mp4 +2025-01-06 20:10:12,713 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:10:12,713 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:10:12,783 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:12,865 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/638dbn2c-media/videos/eval/4_eval_4_7504f2a27cb28e455eb2.mp4 +2025-01-06 20:10:13,799 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:14,385 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:10:14,787 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:15,785 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:16,791 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:17,161 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:10:17,809 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:18,386 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:10:18,793 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:19,798 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:20,387 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:10:20,806 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:21,809 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:22,072 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:10:22,073 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:22,073 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:22,219 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:10:22,219 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:10:22,375 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:10:22,388 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:10:22,388 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:10:22,388 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:10:22,388 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media +2025-01-06 20:10:22,388 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table +2025-01-06 20:10:22,388 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table/eval +2025-01-06 20:10:22,613 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/b1c4gnye-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:10:22,632 INFO wandb-upload_2:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/bpzc5pmd-events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:10:22,768 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:10:22,778 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:10:22,795 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:22,828 INFO wandb-upload_1:2104256 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_9qfkvb_ +2025-01-06 20:10:23,264 INFO SenderThread:2104256 [sender.py:send_request_log_artifact():1525] logged artifact run-m386gyxa-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjIyMjYxOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYwMDI0Mg==', 'latestArtifact': None}} +2025-01-06 20:10:23,264 DEBUG SenderThread:2104256 [sender.py:send():379] send: files +2025-01-06 20:10:23,264 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 20:10:23,265 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:10:23,265 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:10:23,265 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,265 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,266 DEBUG SenderThread:2104256 [sender.py:send():379] send: telemetry +2025-01-06 20:10:23,266 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,269 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,270 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,270 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,270 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,270 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,271 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,271 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,271 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,271 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,272 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,272 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,272 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,272 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,273 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,273 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,273 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,273 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,274 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,274 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,274 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:23,274 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:23,274 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:10:23,388 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:10:23,388 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:10:23,388 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table/eval/total_success_rate_ci_6_6e461524546d8e890233.table.json +2025-01-06 20:10:23,389 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table/eval +2025-01-06 20:10:23,513 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/4jyhaquy-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:10:23,581 INFO wandb-upload_1:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/nwy4c06a-media/table/eval/total_success_rate_ci_6_6e461524546d8e890233.table.json +2025-01-06 20:10:23,632 INFO wandb-upload_2:2104256 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmptzqsyq0c +2025-01-06 20:10:23,795 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:24,000 INFO SenderThread:2104256 [sender.py:send_request_log_artifact():1525] logged artifact run-m386gyxa-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjIyMjcwMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYwMDMwNw==', 'latestArtifact': None}} +2025-01-06 20:10:24,000 DEBUG SenderThread:2104256 [sender.py:send():379] send: files +2025-01-06 20:10:24,000 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_6e461524546d8e890233.table.json with policy now +2025-01-06 20:10:24,000 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:10:24,001 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:24,002 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:24,247 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/0tu7i3cn-media/table/eval/total_success_rate_ci_6_6e461524546d8e890233.table.json +2025-01-06 20:10:24,389 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:10:24,389 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:10:24,795 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:25,795 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:26,795 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:27,366 DEBUG SenderThread:2104256 [sender.py:send():379] send: stats +2025-01-06 20:10:27,705 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:10:27,706 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:10:27,796 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:28,178 DEBUG SenderThread:2104256 [sender.py:send():379] send: exit +2025-01-06 20:10:28,178 INFO SenderThread:2104256 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 20:10:28,178 INFO SenderThread:2104256 [sender.py:send_exit():588] handling runtime: 270 +2025-01-06 20:10:28,179 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:28,179 INFO SenderThread:2104256 [sender.py:send_exit():594] send defer +2025-01-06 20:10:28,179 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:28,179 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 20:10:28,180 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:28,180 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 20:10:28,180 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 1 +2025-01-06 20:10:28,180 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:28,180 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 20:10:28,180 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:28,180 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 20:10:28,180 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 2 +2025-01-06 20:10:28,180 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:28,180 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 20:10:28,180 INFO HandlerThread:2104256 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 20:10:28,180 DEBUG SystemMonitor:2104256 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 20:10:28,180 DEBUG SystemMonitor:2104256 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 20:10:28,181 INFO HandlerThread:2104256 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 20:10:28,181 INFO HandlerThread:2104256 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 20:10:28,233 INFO HandlerThread:2104256 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 20:10:28,233 INFO HandlerThread:2104256 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 20:10:28,233 INFO HandlerThread:2104256 [interfaces.py:finish():200] Joined network monitor +2025-01-06 20:10:28,234 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:28,234 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 20:10:28,234 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 3 +2025-01-06 20:10:28,234 DEBUG SenderThread:2104256 [sender.py:send():379] send: stats +2025-01-06 20:10:28,234 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:28,234 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 20:10:28,234 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:28,234 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 20:10:28,235 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 4 +2025-01-06 20:10:28,235 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:28,235 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 20:10:28,390 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:10:30,391 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:10:35,080 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:10:35,080 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:10:35,080 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 20:10:35,080 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:10:35,081 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:35,081 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 20:10:35,082 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 5 +2025-01-06 20:10:35,082 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:10:35,082 DEBUG SenderThread:2104256 [sender.py:send():379] send: history +2025-01-06 20:10:35,082 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:10:35,082 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:35,082 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 20:10:35,083 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:35,083 DEBUG SenderThread:2104256 [sender.py:send():379] send: summary +2025-01-06 20:10:35,083 INFO SenderThread:2104256 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:10:35,084 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:35,084 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 20:10:35,084 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 6 +2025-01-06 20:10:35,084 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:35,084 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 20:10:35,084 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:35,084 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 20:10:35,087 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:10:35,189 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:10:35,204 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 7 +2025-01-06 20:10:35,204 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:10:35,204 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:35,204 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 20:10:35,205 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:35,205 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 20:10:35,393 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/config.yaml +2025-01-06 20:10:35,393 INFO Thread-12 :2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:10:36,189 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:10:37,993 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 8 +2025-01-06 20:10:37,993 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:10:37,994 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:37,994 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 20:10:37,994 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:37,994 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 20:10:37,994 INFO SenderThread:2104256 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 20:10:37,997 INFO SenderThread:2104256 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 20:10:37,999 INFO SenderThread:2104256 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 20:10:38,008 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 9 +2025-01-06 20:10:38,008 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:38,008 DEBUG SenderThread:2104256 [sender.py:send():379] send: artifact +2025-01-06 20:10:38,008 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 20:10:38,190 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:10:38,224 INFO SenderThread:2104256 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 20:10:38,224 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:38,224 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 20:10:38,224 INFO SenderThread:2104256 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 20:10:38,394 INFO SenderThread:2104256 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:10:38,394 INFO SenderThread:2104256 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files +2025-01-06 20:10:38,395 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/config.yaml config.yaml +2025-01-06 20:10:38,395 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json wandb-summary.json +2025-01-06 20:10:38,395 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/requirements.txt requirements.txt +2025-01-06 20:10:38,395 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-metadata.json wandb-metadata.json +2025-01-06 20:10:38,395 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:10:38,395 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log output.log +2025-01-06 20:10:38,396 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/diff.patch diff.patch +2025-01-06 20:10:38,402 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 20:10:38,402 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/3_eval_3_fd7b6d2ce3d8ed8b882d.mp4 media/videos/eval/3_eval_3_fd7b6d2ce3d8ed8b882d.mp4 +2025-01-06 20:10:38,402 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/4_eval_4_7504f2a27cb28e455eb2.mp4 media/videos/eval/4_eval_4_7504f2a27cb28e455eb2.mp4 +2025-01-06 20:10:38,402 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/0_eval_0_22bed2761bf0b211c58d.mp4 media/videos/eval/0_eval_0_22bed2761bf0b211c58d.mp4 +2025-01-06 20:10:38,402 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/1_eval_1_ef07a287a59b234dfd50.mp4 media/videos/eval/1_eval_1_ef07a287a59b234dfd50.mp4 +2025-01-06 20:10:38,403 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/videos/eval/2_eval_2_e403f7a04c6fe992037c.mp4 media/videos/eval/2_eval_2_e403f7a04c6fe992037c.mp4 +2025-01-06 20:10:38,403 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table/eval/total_success_rate_ci_6_6e461524546d8e890233.table.json media/table/eval/total_success_rate_ci_6_6e461524546d8e890233.table.json +2025-01-06 20:10:38,403 INFO SenderThread:2104256 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:10:38,403 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 10 +2025-01-06 20:10:38,403 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:10:38,403 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:38,403 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 20:10:38,405 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:38,405 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 20:10:38,405 INFO SenderThread:2104256 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:10:38,592 INFO wandb-upload_0:2104256 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/requirements.txt +2025-01-06 20:10:38,599 INFO wandb-upload_2:2104256 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/config.yaml +2025-01-06 20:10:38,617 INFO wandb-upload_4:2104256 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/output.log +2025-01-06 20:10:38,638 INFO wandb-upload_1:2104256 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/files/wandb-summary.json +2025-01-06 20:10:38,684 INFO wandb-upload_3:2104256 [upload_job.py:push():130] Uploaded file /tmp/tmp5r99wq_iwandb/mksmfkw2-events.out.tfevents.1736211957.leonmkim-ROG-Strix-G15CS-G15CS.2104083.0 +2025-01-06 20:10:38,885 INFO Thread-11 (_thread_body):2104256 [sender.py:transition_state():614] send defer: 11 +2025-01-06 20:10:38,885 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:38,885 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 20:10:38,885 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:38,885 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 20:10:38,885 INFO SenderThread:2104256 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 20:10:38,886 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 12 +2025-01-06 20:10:38,886 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:38,887 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 20:10:38,887 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:38,887 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 20:10:38,887 INFO SenderThread:2104256 [file_stream.py:finish():601] file stream finish called +2025-01-06 20:10:38,942 INFO SenderThread:2104256 [file_stream.py:finish():605] file stream finish is done +2025-01-06 20:10:38,943 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 13 +2025-01-06 20:10:38,943 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:38,943 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 20:10:38,943 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:38,943 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 20:10:38,943 INFO SenderThread:2104256 [sender.py:transition_state():614] send defer: 14 +2025-01-06 20:10:38,943 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:10:38,943 INFO HandlerThread:2104256 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 20:10:38,944 DEBUG SenderThread:2104256 [sender.py:send():379] send: final +2025-01-06 20:10:38,944 DEBUG SenderThread:2104256 [sender.py:send():379] send: footer +2025-01-06 20:10:38,944 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: defer +2025-01-06 20:10:38,944 INFO SenderThread:2104256 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 20:10:38,944 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:10:38,944 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:10:38,944 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 20:10:38,945 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: server_info +2025-01-06 20:10:38,945 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:10:38,946 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 20:10:38,947 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 20:10:38,947 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:10:38,992 DEBUG SenderThread:2104256 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:10:38,993 INFO MainThread:2104256 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 20:10:38,993 INFO MainThread:2104256 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 20:10:38,993 INFO MainThread:2104256 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 20:10:38,994 DEBUG HandlerThread:2104256 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 20:10:38,994 INFO HandlerThread:2104256 [handler.py:finish():882] shutting down handler +2025-01-06 20:10:39,947 INFO WriterThread:2104256 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/run-m386gyxa.wandb +2025-01-06 20:10:39,993 INFO SenderThread:2104256 [sender.py:finish():1615] shutting down sender +2025-01-06 20:10:39,993 INFO SenderThread:2104256 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:10:39,993 INFO SenderThread:2104256 [file_pusher.py:join():175] waiting for file pusher diff --git a/m386gyxa/wandb/run-20250106_200557-m386gyxa/logs/debug.log b/m386gyxa/wandb/run-20250106_200557-m386gyxa/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..be44991f25d7fcd08eee1a76f1eea6bd52734196 --- /dev/null +++ b/m386gyxa/wandb/run-20250106_200557-m386gyxa/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 20:05:57,038 INFO MainThread:2104083 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 20:05:57,038 INFO MainThread:2104083 [wandb_setup.py:_flush():76] Configure stats pid to 2104083 +2025-01-06 20:05:57,038 INFO MainThread:2104083 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 20:05:57,039 INFO MainThread:2104083 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/settings +2025-01-06 20:05:57,039 INFO MainThread:2104083 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 20:05:57,039 INFO MainThread:2104083 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 20:05:57,039 INFO MainThread:2104083 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 20:05:57,039 INFO MainThread:2104083 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 20:05:57,039 INFO MainThread:2104083 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/logs/debug.log +2025-01-06 20:05:57,039 INFO MainThread:2104083 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/wandb/run-20250106_200557-m386gyxa/logs/debug-internal.log +2025-01-06 20:05:57,039 INFO MainThread:2104083 [wandb_init.py:init():569] calling init triggers +2025-01-06 20:05:57,040 INFO MainThread:2104083 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa'} +2025-01-06 20:05:57,040 INFO MainThread:2104083 [wandb_init.py:init():619] starting backend +2025-01-06 20:05:57,040 INFO MainThread:2104083 [wandb_init.py:init():623] setting up manager +2025-01-06 20:05:57,044 INFO MainThread:2104083 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 20:05:57,045 INFO MainThread:2104083 [wandb_init.py:init():631] backend started and connected +2025-01-06 20:05:57,059 INFO MainThread:2104083 [wandb_init.py:init():720] updated telemetry +2025-01-06 20:05:57,067 INFO MainThread:2104083 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 20:05:57,308 INFO MainThread:2104083 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 20:05:57,347 INFO MainThread:2104083 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 20:05:57,348 INFO MainThread:2104083 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 20:05:57,700 INFO MainThread:2104083 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 20:05:57,701 INFO MainThread:2104083 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 20:05:57,701 INFO MainThread:2104083 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 20:05:57,701 INFO MainThread:2104083 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 20:05:57,702 INFO MainThread:2104083 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 20:05:57,702 INFO MainThread:2104083 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/m386gyxa/tb, True +2025-01-06 20:06:02,408 INFO MainThread:2104083 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-06 20:10:39,995 WARNING MsgRouterThr:2104083 [router.py:message_loop():77] message_loop has been closed diff --git a/m386gyxa/wandb/run-20250106_200557-m386gyxa/run-m386gyxa.wandb b/m386gyxa/wandb/run-20250106_200557-m386gyxa/run-m386gyxa.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d806060e69fac7ff00d56d6928a877a8798cd6d6 Binary files /dev/null and b/m386gyxa/wandb/run-20250106_200557-m386gyxa/run-m386gyxa.wandb differ diff --git a/up4gjipz/.hydra/config.yaml b/up4gjipz/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd5530cb9a8f25d6d55135607d04c2061b4351d1 --- /dev/null +++ b/up4gjipz/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1017_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/up4gjipz/.hydra/hydra.yaml b/up4gjipz/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb9010bab1ca425152d7bcf83ecdd7e59c198771 --- /dev/null +++ b/up4gjipz/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/up4gjipz + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/up4gjipz/.hydra/overrides.yaml b/up4gjipz/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/up4gjipz/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/up4gjipz/episode_rosbags/episode_3_2025-01-06-20-03-53.bag b/up4gjipz/episode_rosbags/episode_3_2025-01-06-20-03-53.bag new file mode 100644 index 0000000000000000000000000000000000000000..b18d61e3be89ee513514f8e10f2dd3055ec2022b --- /dev/null +++ b/up4gjipz/episode_rosbags/episode_3_2025-01-06-20-03-53.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9178367e67f93d66766f28eb0424289823b33206ea69e79309377ee127ba5370 +size 2603164593 diff --git a/up4gjipz/eval_video/0_eval.mp4 b/up4gjipz/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..35e38a55d5d6e52691831b11ed048160d3130a16 --- /dev/null +++ b/up4gjipz/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10ebed5c632009219ad7d9302f7f5a63e08a512951d50a98abbf25a2373cb531 +size 1449386 diff --git a/up4gjipz/eval_video/1_eval.mp4 b/up4gjipz/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0a3e840b5d25f7af228414fb5199d6d980a78666 --- /dev/null +++ b/up4gjipz/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb1fe7858735fb8913041554892c4674ef9522fc367fbfd3a84fc79cccefb279 +size 1449250 diff --git a/up4gjipz/eval_video/2_eval.mp4 b/up4gjipz/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d6c761f098cdd493763a41258b08398b4f6ba96d --- /dev/null +++ b/up4gjipz/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2afb9a1beef18459027384888451a24b2d1097337e11c58f263d8f2652da5473 +size 1459030 diff --git a/up4gjipz/eval_video/3_eval.mp4 b/up4gjipz/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9d1733d0f067593f7d93241c1c8cd424c8d63621 --- /dev/null +++ b/up4gjipz/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f13de8c1cf058ceccfd9f7d89923048c5fe77f56744140af762e4a52a063fb4 +size 1441720 diff --git a/up4gjipz/eval_video/4_eval.mp4 b/up4gjipz/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fe0d293fa6f5a1bd897feeb21556da52def51d39 --- /dev/null +++ b/up4gjipz/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d921c57a920655085eff556536a59a60176aadeb38e3fd2e879763debea3738 +size 1442675 diff --git a/up4gjipz/tb/events.out.tfevents.1736211657.leonmkim-ROG-Strix-G15CS-G15CS.2103006.0 b/up4gjipz/tb/events.out.tfevents.1736211657.leonmkim-ROG-Strix-G15CS-G15CS.2103006.0 new file mode 100644 index 0000000000000000000000000000000000000000..b7fb92fcc55f6301e388919ad560ed3de4563e03 --- /dev/null +++ b/up4gjipz/tb/events.out.tfevents.1736211657.leonmkim-ROG-Strix-G15CS-G15CS.2103006.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b246f14dd42b5da86c6cf72262cff04f7b41962b832f20b25b9c8c7377662c9d +size 6539 diff --git a/urh96fgp/episode_rosbags/episode_0_2025-01-07-20-11-41.bag b/urh96fgp/episode_rosbags/episode_0_2025-01-07-20-11-41.bag new file mode 100644 index 0000000000000000000000000000000000000000..bf0155a90d0261ca0372a297678343a5d22b6c87 --- /dev/null +++ b/urh96fgp/episode_rosbags/episode_0_2025-01-07-20-11-41.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33f50d30a0b8110e699ec1d4d2f239523f26ed5ad83d41d8cc707c015da39d4d +size 1970297005 diff --git a/urh96fgp/episode_rosbags/episode_1_2025-01-07-20-12-33.bag b/urh96fgp/episode_rosbags/episode_1_2025-01-07-20-12-33.bag new file mode 100644 index 0000000000000000000000000000000000000000..a4b9c483475a795882dde30b05bb99979dbfe4ae --- /dev/null +++ b/urh96fgp/episode_rosbags/episode_1_2025-01-07-20-12-33.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f3ad853e1edd55d0b857be064e56bf263365a1323bbdd08a46dcbbdf62076d0 +size 1922842494 diff --git a/x85ya2ao/.hydra/config.yaml b/x85ya2ao/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd5530cb9a8f25d6d55135607d04c2061b4351d1 --- /dev/null +++ b/x85ya2ao/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1017_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/x85ya2ao/.hydra/hydra.yaml b/x85ya2ao/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1cac5eed7482b25287fef6445922c465630a16a0 --- /dev/null +++ b/x85ya2ao/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/x85ya2ao/.hydra/overrides.yaml b/x85ya2ao/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/x85ya2ao/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/x85ya2ao/episode_rosbags/aligned_depth_to_color_K.npy b/x85ya2ao/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/x85ya2ao/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/x85ya2ao/episode_rosbags/cam_tf_world.npy b/x85ya2ao/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/x85ya2ao/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/x85ya2ao/episode_rosbags/color_K.npy b/x85ya2ao/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/x85ya2ao/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/x85ya2ao/episode_rosbags/depth_K.npy b/x85ya2ao/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/x85ya2ao/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/x85ya2ao/eval_robot.log b/x85ya2ao/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..92d8333a6e5d2abe14725c85fe7ae47d21f5b33b --- /dev/null +++ b/x85ya2ao/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 18:28:23,463][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 18:28:23,466][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/x85ya2ao/eval_video/0_eval.mp4 b/x85ya2ao/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1d1b66271f2a40dc6dcbde6afed22a8155571baa --- /dev/null +++ b/x85ya2ao/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ad56c694ef7147e5404200e1d7187387e234f4fbe226b1ead59c0c7110db96d +size 1451975 diff --git a/x85ya2ao/eval_video/1_eval.mp4 b/x85ya2ao/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..78a1516ace90208a1b4d0dff9aa4f41cc5e36efe --- /dev/null +++ b/x85ya2ao/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c12723790e5302e09cecfad97ad8bed87763a0f20764126ee63807b24aabed94 +size 1430855 diff --git a/x85ya2ao/eval_video/2_eval.mp4 b/x85ya2ao/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68fd3f0d5250c6a457a86ff271b98c193b321ae9 --- /dev/null +++ b/x85ya2ao/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85ba534ec05635d7199667d49fd66950ba507f4b70eaa3430428a01b705eeee2 +size 1433745 diff --git a/x85ya2ao/eval_video/3_eval.mp4 b/x85ya2ao/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2028206ca374a35c8ecb333ab819c67efd433e63 --- /dev/null +++ b/x85ya2ao/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88082439ff7f6a045e2d1cbfbebbf279576bc2d88a9501160a136b0d0ee7835b +size 1434700 diff --git a/x85ya2ao/eval_video/4_eval.mp4 b/x85ya2ao/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..090b6ffd3506210a33ae4853670ca76579a47d48 --- /dev/null +++ b/x85ya2ao/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df2f032446cb0a313989504722ba4794f92400a01508c67b78cf85f246b1301a +size 1432052 diff --git a/x85ya2ao/tb/events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 b/x85ya2ao/tb/events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 new file mode 100644 index 0000000000000000000000000000000000000000..9871881784ef8e1b6e686584b2dae9ab6e6dae2c --- /dev/null +++ b/x85ya2ao/tb/events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09054ba3d849a855e7805e09024fcb5af71322fa1ad83c012b3fa13fd8962174 +size 6539 diff --git a/x85ya2ao/wandb/debug-internal.log b/x85ya2ao/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..609942e351027aa009e8dca9dc42d81223633d5c --- /dev/null +++ b/x85ya2ao/wandb/debug-internal.log @@ -0,0 +1,769 @@ +2025-01-07 18:28:29,351 INFO StreamThr :2213249 [internal.py:wandb_internal():85] W&B internal server running at pid: 2213249, started at: 2025-01-07 18:28:29.350862 +2025-01-07 18:28:29,352 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status +2025-01-07 18:28:29,364 INFO WriterThread:2213249 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/run-x85ya2ao.wandb +2025-01-07 18:28:29,364 DEBUG SenderThread:2213249 [sender.py:send():379] send: header +2025-01-07 18:28:29,371 DEBUG SenderThread:2213249 [sender.py:send():379] send: run +2025-01-07 18:28:29,623 INFO SenderThread:2213249 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files +2025-01-07 18:28:29,623 INFO SenderThread:2213249 [sender.py:_start_run_threads():1188] run started: x85ya2ao with start time 1736292509.351827 +2025-01-07 18:28:29,637 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 18:28:29,637 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: check_version +2025-01-07 18:28:29,710 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 18:28:29,714 DEBUG HandlerThread:2213249 [system_info.py:__init__():26] System info init +2025-01-07 18:28:29,714 DEBUG HandlerThread:2213249 [system_info.py:__init__():41] System info init done +2025-01-07 18:28:29,714 INFO HandlerThread:2213249 [system_monitor.py:start():194] Starting system monitor +2025-01-07 18:28:29,715 INFO SystemMonitor:2213249 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 18:28:29,715 INFO HandlerThread:2213249 [system_monitor.py:probe():214] Collecting system info +2025-01-07 18:28:29,715 INFO SystemMonitor:2213249 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 18:28:29,716 INFO SystemMonitor:2213249 [interfaces.py:start():188] Started disk monitoring +2025-01-07 18:28:29,716 INFO SystemMonitor:2213249 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 18:28:29,717 INFO SystemMonitor:2213249 [interfaces.py:start():188] Started memory monitoring +2025-01-07 18:28:29,717 INFO SystemMonitor:2213249 [interfaces.py:start():188] Started network monitoring +2025-01-07 18:28:29,728 DEBUG HandlerThread:2213249 [system_info.py:probe():152] Probing system +2025-01-07 18:28:29,730 DEBUG HandlerThread:2213249 [system_info.py:_probe_git():137] Probing git +2025-01-07 18:28:29,734 DEBUG HandlerThread:2213249 [system_info.py:_probe_git():145] Probing git done +2025-01-07 18:28:29,734 DEBUG HandlerThread:2213249 [system_info.py:probe():200] Probing system done +2025-01-07 18:28:29,734 DEBUG HandlerThread:2213249 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T23:28:29.728596', 'startedAt': '2025-01-07T23:28:29.340605', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.709375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.97, 'min': 800.0, 'max': 4700.0}, {'current': 4496.86, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 850.1733322143555}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 18:28:29,734 INFO HandlerThread:2213249 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 18:28:29,734 INFO HandlerThread:2213249 [system_monitor.py:probe():227] Publishing system info +2025-01-07 18:28:29,734 DEBUG HandlerThread:2213249 [system_info.py:_save_code():44] Saving code +2025-01-07 18:28:29,736 DEBUG HandlerThread:2213249 [system_info.py:_save_code():67] Saving code done +2025-01-07 18:28:29,736 DEBUG HandlerThread:2213249 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 18:28:29,776 DEBUG HandlerThread:2213249 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 18:28:29,776 INFO HandlerThread:2213249 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 18:28:29,780 DEBUG SenderThread:2213249 [sender.py:send():379] send: files +2025-01-07 18:28:29,780 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 18:28:29,780 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 18:28:30,018 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 18:28:30,019 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: python_packages +2025-01-07 18:28:30,019 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:30,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:28:30,020 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:28:30,023 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/1a96eit6-wandb-metadata.json +2025-01-07 18:28:30,061 INFO HandlerThread:2213249 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/tb" + save: true +} +control { + relay_id: "4cbe04b595d84eb49319b193420ac055" +} +_info { + stream_id: "x85ya2ao" +} + +2025-01-07 18:28:30,061 DEBUG HandlerThread:2213249 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 18:28:30,068 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/qxdyx2pg-code/FISH/eval_robot.py +2025-01-07 18:28:30,152 DEBUG SenderThread:2213249 [sender.py:send():379] send: telemetry +2025-01-07 18:28:30,152 DEBUG SenderThread:2213249 [sender.py:send():379] send: tbrecord +2025-01-07 18:28:30,153 DEBUG SenderThread:2213249 [sender.py:send():379] send: files +2025-01-07 18:28:30,153 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 with policy live +2025-01-07 18:28:30,326 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/kteafxg6-events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:28:30,625 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/requirements.txt +2025-01-07 18:28:30,625 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:28:30,625 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:28:30,625 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/code/FISH/eval_robot.py +2025-01-07 18:28:30,625 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-metadata.json +2025-01-07 18:28:30,625 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/code +2025-01-07 18:28:30,625 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/code/FISH +2025-01-07 18:28:31,019 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:32,019 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:32,624 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:28:33,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:34,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:35,019 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:28:35,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:36,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:37,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:38,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:38,363 DEBUG SenderThread:2213249 [sender.py:send():379] send: config +2025-01-07 18:28:39,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:40,019 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:28:40,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:40,626 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:28:41,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:42,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:43,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:44,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:44,627 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:28:45,019 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:28:45,019 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:28:45,061 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:45,088 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:28:46,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:47,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:48,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:48,628 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:28:49,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:50,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:50,841 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:28:51,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:52,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:52,629 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:28:53,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:54,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:54,630 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:28:55,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:56,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:56,631 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:28:56,721 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:28:57,023 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:58,023 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:59,023 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:00,019 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:29:00,019 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:29:00,065 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:01,024 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:02,024 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:02,632 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:29:02,717 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:03,024 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:03,632 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/config.yaml +2025-01-07 18:29:04,024 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:05,024 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:06,024 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:07,025 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:08,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:08,025 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:09,025 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:10,025 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:11,025 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:12,025 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:13,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:13,025 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:14,025 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:15,019 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:29:15,019 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:29:15,065 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:16,026 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:17,026 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:18,026 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:18,092 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:19,026 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:20,026 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:21,026 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:22,027 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:23,027 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:24,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:24,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:25,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:26,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:27,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:28,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:29,031 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:29,430 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:29,639 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/0_eval_0_7ad56c694ef7147e5404.mp4 +2025-01-07 18:29:29,639 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval +2025-01-07 18:29:29,639 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media +2025-01-07 18:29:29,639 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos +2025-01-07 18:29:29,642 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:29:29,642 DEBUG SenderThread:2213249 [sender.py:send():379] send: files +2025-01-07 18:29:29,643 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_7ad56c694ef7147e5404.mp4 with policy now +2025-01-07 18:29:29,643 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:29:29,643 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:29:29,644 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:29:29,717 DEBUG SystemMonitor:2213249 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 18:29:29,718 DEBUG SenderThread:2213249 [sender.py:send():379] send: stats +2025-01-07 18:29:29,910 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/h1oef31b-media/videos/eval/0_eval_0_7ad56c694ef7147e5404.mp4 +2025-01-07 18:29:30,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:29:30,020 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:29:30,030 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:30,226 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/f1ypq05s-media/videos/eval/0_eval_0_7ad56c694ef7147e5404.mp4 +2025-01-07 18:29:30,639 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:29:30,639 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:29:31,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:32,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:33,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:34,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:34,640 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:29:35,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:35,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:36,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:37,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:37,642 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:29:37,897 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/h6jgbzsy-events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:29:38,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:38,642 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:29:39,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:40,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:40,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:40,643 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:29:41,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:42,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:42,643 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:29:43,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:44,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:45,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:29:45,020 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:29:45,061 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:45,112 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:46,030 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:47,031 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:48,031 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:49,032 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:50,031 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:50,646 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:29:51,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:51,032 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:52,033 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:53,032 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:54,032 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:55,032 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:56,033 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:57,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:57,033 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:58,033 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:59,033 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:59,718 DEBUG SenderThread:2213249 [sender.py:send():379] send: stats +2025-01-07 18:30:00,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:30:00,020 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:30:00,061 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:01,034 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:02,034 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:02,178 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:03,034 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:04,034 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:05,035 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:06,035 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:07,035 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:08,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:08,035 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:09,035 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:10,036 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:11,036 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:12,036 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:13,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:13,036 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:14,036 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:15,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:30:15,020 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:30:15,065 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:16,036 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:17,036 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:17,350 DEBUG SenderThread:2213249 [sender.py:send():379] send: files +2025-01-07 18:30:17,350 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_c12723790e5302e09cec.mp4 with policy now +2025-01-07 18:30:17,351 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:30:17,351 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:30:17,351 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:30:17,355 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:30:17,615 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/ls47xbfn-media/videos/eval/1_eval_1_c12723790e5302e09cec.mp4 +2025-01-07 18:30:17,654 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:30:17,654 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/1_eval_1_c12723790e5302e09cec.mp4 +2025-01-07 18:30:17,655 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval +2025-01-07 18:30:17,941 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/vex9i1rp-media/videos/eval/1_eval_1_c12723790e5302e09cec.mp4 +2025-01-07 18:30:18,038 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:18,372 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:18,655 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:30:19,042 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:19,655 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:30:20,044 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:21,040 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:22,040 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:22,656 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:30:23,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:23,876 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:24,040 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:25,040 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:26,040 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:26,657 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:30:27,040 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:27,489 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:30:27,489 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:30:27,489 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:30:27,658 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:30:27,658 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:30:28,044 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:28,658 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:30:29,042 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:29,205 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:29,719 DEBUG SenderThread:2213249 [sender.py:send():379] send: stats +2025-01-07 18:30:30,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:30:30,020 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:30:30,061 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:30,659 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:30:31,043 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:32,043 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:32,660 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:30:33,043 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:34,043 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:35,023 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:35,043 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:36,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:37,044 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:38,044 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:39,044 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:40,044 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:40,455 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:41,044 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:42,044 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:42,663 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:30:43,044 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:44,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:45,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:30:45,020 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:30:45,061 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:46,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:46,137 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:47,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:48,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:49,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:50,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:51,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:51,389 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:52,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:52,666 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:30:53,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:54,046 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:55,046 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:56,046 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:57,024 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:57,047 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:58,046 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:59,046 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:59,720 DEBUG SenderThread:2213249 [sender.py:send():379] send: stats +2025-01-07 18:31:00,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:31:00,021 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:31:00,061 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:01,047 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:02,047 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:02,087 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:03,047 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:04,047 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:05,047 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:06,048 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:06,671 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:31:06,826 DEBUG SenderThread:2213249 [sender.py:send():379] send: files +2025-01-07 18:31:06,826 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_85ba534ec05635d71996.mp4 with policy now +2025-01-07 18:31:06,831 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:31:06,832 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:31:06,832 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:31:06,833 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:31:07,048 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:07,091 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/zd0gdtxj-media/videos/eval/2_eval_2_85ba534ec05635d71996.mp4 +2025-01-07 18:31:07,671 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:31:07,671 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/2_eval_2_85ba534ec05635d71996.mp4 +2025-01-07 18:31:07,672 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval +2025-01-07 18:31:07,865 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:07,967 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/03fwnkzc-media/videos/eval/2_eval_2_85ba534ec05635d71996.mp4 +2025-01-07 18:31:08,049 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:08,672 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:31:09,053 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:10,059 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:11,055 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:12,061 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:12,673 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:31:13,060 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:13,350 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:14,060 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:14,674 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:31:15,025 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:31:15,025 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:31:15,069 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:16,064 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:17,064 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:18,065 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:19,070 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:19,124 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:20,078 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:21,068 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:22,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:23,070 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:23,316 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:31:23,317 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:31:23,318 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:31:23,677 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:31:23,677 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:31:23,948 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/zrriz22a-events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:31:24,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:24,294 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:24,677 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:31:25,075 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:26,078 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:26,678 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:31:27,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:28,072 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:28,678 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:31:29,072 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:29,363 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:29,720 DEBUG SenderThread:2213249 [sender.py:send():379] send: stats +2025-01-07 18:31:30,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:31:30,021 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:31:30,072 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:31,072 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:32,072 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:33,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:34,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:35,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:35,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:36,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:37,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:38,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:39,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:40,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:40,075 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:41,075 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:42,076 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:43,075 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:44,076 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:45,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:31:45,021 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:31:45,076 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:45,092 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:46,076 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:47,076 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:48,076 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:49,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:50,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:51,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:51,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:52,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:53,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:54,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:55,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:56,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:56,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:57,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:58,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:59,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:59,720 DEBUG SenderThread:2213249 [sender.py:send():379] send: stats +2025-01-07 18:32:00,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:32:00,021 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:32:00,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:01,078 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:01,145 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:02,078 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:03,088 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:03,185 DEBUG SenderThread:2213249 [sender.py:send():379] send: files +2025-01-07 18:32:03,185 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_88082439ff7f6a045e2d.mp4 with policy now +2025-01-07 18:32:03,195 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:32:03,196 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:32:03,196 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:32:03,197 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:32:03,465 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/urcslq2m-media/videos/eval/3_eval_3_88082439ff7f6a045e2d.mp4 +2025-01-07 18:32:03,690 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:32:03,690 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/3_eval_3_88082439ff7f6a045e2d.mp4 +2025-01-07 18:32:03,690 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval +2025-01-07 18:32:03,982 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/3ycc1n9w-media/videos/eval/3_eval_3_88082439ff7f6a045e2d.mp4 +2025-01-07 18:32:04,090 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:04,690 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:05,093 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:06,090 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:06,947 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:07,095 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:08,100 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:08,692 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:09,108 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:10,092 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:10,692 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:11,093 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:11,372 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:32:11,373 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:32:11,373 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:32:11,693 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:32:11,693 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:32:12,097 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:12,693 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:12,877 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:13,098 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:14,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:15,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:32:15,029 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:32:15,102 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:16,102 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:16,695 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:17,102 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:18,102 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:18,127 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:19,103 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:20,103 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:21,103 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:22,103 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:23,103 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:24,033 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:24,103 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:25,103 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:26,103 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:27,103 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:28,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:29,034 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:29,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:29,721 DEBUG SenderThread:2213249 [sender.py:send():379] send: stats +2025-01-07 18:32:30,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:32:30,021 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:32:30,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:31,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:32,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:33,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:34,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:34,176 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:35,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:36,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:37,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:38,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:39,105 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:40,033 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:40,105 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:41,105 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:42,105 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:43,105 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:44,105 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:45,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:32:45,022 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:32:45,088 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:45,105 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:46,106 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:47,106 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:48,106 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:49,106 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:50,117 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:50,329 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:50,706 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:51,096 DEBUG SenderThread:2213249 [sender.py:send():379] send: files +2025-01-07 18:32:51,096 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_df2f032446cb0a313989.mp4 with policy now +2025-01-07 18:32:51,099 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:32:51,099 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:32:51,099 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:32:51,100 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:32:51,119 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:51,378 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/mbdgva40-media/videos/eval/4_eval_4_df2f032446cb0a313989.mp4 +2025-01-07 18:32:51,706 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:32:51,707 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/4_eval_4_df2f032446cb0a313989.mp4 +2025-01-07 18:32:51,707 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval +2025-01-07 18:32:51,990 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/5q4z2he1-media/videos/eval/4_eval_4_df2f032446cb0a313989.mp4 +2025-01-07 18:32:52,115 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:52,707 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:53,120 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:53,707 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:54,121 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:55,125 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:56,039 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:56,121 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:56,708 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:57,124 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:58,129 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:58,709 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:59,132 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:59,432 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:32:59,433 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:32:59,433 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:32:59,596 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 18:32:59,596 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 18:32:59,709 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:32:59,709 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:32:59,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:32:59,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media +2025-01-07 18:32:59,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table/eval +2025-01-07 18:32:59,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table +2025-01-07 18:32:59,830 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:32:59,941 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/c2edweux-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:32:59,988 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/qhx6jioi-events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:33:00,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:33:00,068 INFO wandb-upload_2:2213249 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzr9fqxos +2025-01-07 18:33:00,130 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:00,235 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 18:33:00,244 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:33:00,491 INFO SenderThread:2213249 [sender.py:send_request_log_artifact():1525] logged artifact run-x85ya2ao-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjEyNzIyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTc2MDEzOA==', 'latestArtifact': None}} +2025-01-07 18:33:00,491 DEBUG SenderThread:2213249 [sender.py:send():379] send: files +2025-01-07 18:33:00,492 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-07 18:33:00,492 DEBUG SenderThread:2213249 [sender.py:send():379] send: stats +2025-01-07 18:33:00,492 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:33:00,492 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,492 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,493 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:33:00,541 DEBUG SenderThread:2213249 [sender.py:send():379] send: telemetry +2025-01-07 18:33:00,542 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,542 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,542 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,542 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,542 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,543 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,543 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,543 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,543 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,544 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,544 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,545 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,545 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,545 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,546 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,546 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,546 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,546 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,546 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,547 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,547 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,547 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,547 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 18:33:00,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:33:00,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:33:00,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:33:00,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table/eval/total_success_rate_ci_6_820e6488246e96d3e291.table.json +2025-01-07 18:33:00,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table/eval +2025-01-07 18:33:00,747 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/6sq8pmjf-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:33:00,897 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/vbpq42kf-media/table/eval/total_success_rate_ci_6_820e6488246e96d3e291.table.json +2025-01-07 18:33:00,923 INFO wandb-upload_2:2213249 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0qw9bmm0 +2025-01-07 18:33:01,130 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:01,338 INFO SenderThread:2213249 [sender.py:send_request_log_artifact():1525] logged artifact run-x85ya2ao-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjEyNzI1OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTc2MDE0Mg==', 'latestArtifact': None}} +2025-01-07 18:33:01,338 DEBUG SenderThread:2213249 [sender.py:send():379] send: files +2025-01-07 18:33:01,338 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_820e6488246e96d3e291.table.json with policy now +2025-01-07 18:33:01,338 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:33:01,339 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:33:01,339 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:01,339 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:01,601 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/mojljgve-media/table/eval/total_success_rate_ci_6_820e6488246e96d3e291.table.json +2025-01-07 18:33:01,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:33:01,746 DEBUG SenderThread:2213249 [sender.py:send():379] send: exit +2025-01-07 18:33:01,746 INFO SenderThread:2213249 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 18:33:01,746 INFO SenderThread:2213249 [sender.py:send_exit():588] handling runtime: 272 +2025-01-07 18:33:01,746 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:01,746 INFO SenderThread:2213249 [sender.py:send_exit():594] send defer +2025-01-07 18:33:01,746 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:01,746 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 18:33:01,747 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:01,747 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 18:33:01,747 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 1 +2025-01-07 18:33:01,747 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:01,747 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 18:33:01,747 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:01,747 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 18:33:01,747 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 2 +2025-01-07 18:33:01,747 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:01,747 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 18:33:01,748 INFO HandlerThread:2213249 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 18:33:01,748 DEBUG SystemMonitor:2213249 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 18:33:01,748 DEBUG SystemMonitor:2213249 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 18:33:01,748 INFO HandlerThread:2213249 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 18:33:01,749 INFO HandlerThread:2213249 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 18:33:01,801 INFO HandlerThread:2213249 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 18:33:01,802 INFO HandlerThread:2213249 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 18:33:01,802 INFO HandlerThread:2213249 [interfaces.py:finish():200] Joined network monitor +2025-01-07 18:33:01,802 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:01,802 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 18:33:01,802 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 3 +2025-01-07 18:33:01,802 DEBUG SenderThread:2213249 [sender.py:send():379] send: stats +2025-01-07 18:33:01,803 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:01,803 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 18:33:01,803 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:01,803 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 18:33:01,803 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 4 +2025-01-07 18:33:01,803 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:01,803 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 18:33:02,711 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:33:02,711 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:33:08,436 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:33:08,436 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:08,436 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:33:08,436 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 18:33:08,436 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 18:33:08,436 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 5 +2025-01-07 18:33:08,437 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:33:08,437 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:08,438 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 18:33:08,439 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:33:08,439 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:08,439 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:08,440 DEBUG SenderThread:2213249 [sender.py:send():379] send: summary +2025-01-07 18:33:08,440 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:08,440 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:08,440 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 18:33:08,440 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 6 +2025-01-07 18:33:08,440 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:08,440 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 18:33:08,440 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:08,440 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 18:33:08,444 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:33:08,540 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 7 +2025-01-07 18:33:08,540 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:08,540 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 18:33:08,540 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:08,540 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 18:33:08,713 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/config.yaml +2025-01-07 18:33:08,713 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:33:08,747 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:33:11,349 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 8 +2025-01-07 18:33:11,349 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:33:11,349 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:11,349 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 18:33:11,350 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:11,350 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 18:33:11,350 INFO SenderThread:2213249 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 18:33:11,352 INFO SenderThread:2213249 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 18:33:11,354 INFO SenderThread:2213249 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 18:33:11,366 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 9 +2025-01-07 18:33:11,366 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:11,366 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 18:33:11,366 DEBUG SenderThread:2213249 [sender.py:send():379] send: artifact +2025-01-07 18:33:11,561 INFO SenderThread:2213249 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTc1Nzg0Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 18:33:11,561 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:11,561 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 18:33:11,561 INFO SenderThread:2213249 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 18:33:11,714 INFO SenderThread:2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:33:11,715 INFO SenderThread:2213249 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files +2025-01-07 18:33:11,715 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/config.yaml config.yaml +2025-01-07 18:33:11,715 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json wandb-summary.json +2025-01-07 18:33:11,715 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/requirements.txt requirements.txt +2025-01-07 18:33:11,715 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-metadata.json wandb-metadata.json +2025-01-07 18:33:11,715 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:33:11,717 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log output.log +2025-01-07 18:33:11,719 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 18:33:11,720 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/3_eval_3_88082439ff7f6a045e2d.mp4 media/videos/eval/3_eval_3_88082439ff7f6a045e2d.mp4 +2025-01-07 18:33:11,720 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/2_eval_2_85ba534ec05635d71996.mp4 media/videos/eval/2_eval_2_85ba534ec05635d71996.mp4 +2025-01-07 18:33:11,720 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/1_eval_1_c12723790e5302e09cec.mp4 media/videos/eval/1_eval_1_c12723790e5302e09cec.mp4 +2025-01-07 18:33:11,720 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/0_eval_0_7ad56c694ef7147e5404.mp4 media/videos/eval/0_eval_0_7ad56c694ef7147e5404.mp4 +2025-01-07 18:33:11,720 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/4_eval_4_df2f032446cb0a313989.mp4 media/videos/eval/4_eval_4_df2f032446cb0a313989.mp4 +2025-01-07 18:33:11,721 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table/eval/total_success_rate_ci_6_820e6488246e96d3e291.table.json media/table/eval/total_success_rate_ci_6_820e6488246e96d3e291.table.json +2025-01-07 18:33:11,721 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:33:11,721 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 10 +2025-01-07 18:33:11,721 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:11,722 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 18:33:11,724 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:11,724 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 18:33:11,724 INFO SenderThread:2213249 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:33:11,748 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:33:11,748 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:33:11,895 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:33:11,916 INFO wandb-upload_2:2213249 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/config.yaml +2025-01-07 18:33:11,919 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/requirements.txt +2025-01-07 18:33:12,010 INFO wandb-upload_4:2213249 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:33:12,031 INFO wandb-upload_3:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/stpjwhdl-events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:33:12,231 INFO Thread-11 (_thread_body):2213249 [sender.py:transition_state():614] send defer: 11 +2025-01-07 18:33:12,231 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:12,231 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 18:33:12,231 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:12,231 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 18:33:12,232 INFO SenderThread:2213249 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 18:33:12,233 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 12 +2025-01-07 18:33:12,233 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:12,233 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 18:33:12,233 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:12,233 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 18:33:12,233 INFO SenderThread:2213249 [file_stream.py:finish():601] file stream finish called +2025-01-07 18:33:12,325 INFO SenderThread:2213249 [file_stream.py:finish():605] file stream finish is done +2025-01-07 18:33:12,325 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 13 +2025-01-07 18:33:12,325 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:12,325 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 18:33:12,325 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:12,326 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 18:33:12,326 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 14 +2025-01-07 18:33:12,326 DEBUG SenderThread:2213249 [sender.py:send():379] send: final +2025-01-07 18:33:12,326 DEBUG SenderThread:2213249 [sender.py:send():379] send: footer +2025-01-07 18:33:12,326 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:12,326 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 18:33:12,326 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:12,326 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 18:33:12,327 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:33:12,327 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:33:12,327 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 18:33:12,327 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:33:12,328 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: server_info +2025-01-07 18:33:12,328 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 18:33:12,329 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 18:33:12,330 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:12,380 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:33:12,380 INFO MainThread:2213249 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 18:33:12,381 INFO MainThread:2213249 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 18:33:12,381 INFO MainThread:2213249 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 18:33:12,381 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 18:33:12,381 INFO HandlerThread:2213249 [handler.py:finish():882] shutting down handler +2025-01-07 18:33:13,328 INFO WriterThread:2213249 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/run-x85ya2ao.wandb +2025-01-07 18:33:13,380 INFO SenderThread:2213249 [sender.py:finish():1615] shutting down sender +2025-01-07 18:33:13,380 INFO SenderThread:2213249 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:33:13,380 INFO SenderThread:2213249 [file_pusher.py:join():175] waiting for file pusher diff --git a/x85ya2ao/wandb/debug.log b/x85ya2ao/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8cd48ed072ffad3061da06728c6842a66ee24bf2 --- /dev/null +++ b/x85ya2ao/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 18:28:29,343 INFO MainThread:2213050 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 18:28:29,343 INFO MainThread:2213050 [wandb_setup.py:_flush():76] Configure stats pid to 2213050 +2025-01-07 18:28:29,343 INFO MainThread:2213050 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 18:28:29,343 INFO MainThread:2213050 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/settings +2025-01-07 18:28:29,344 INFO MainThread:2213050 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 18:28:29,344 INFO MainThread:2213050 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 18:28:29,344 INFO MainThread:2213050 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 18:28:29,344 INFO MainThread:2213050 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 18:28:29,344 INFO MainThread:2213050 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/logs/debug.log +2025-01-07 18:28:29,344 INFO MainThread:2213050 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/logs/debug-internal.log +2025-01-07 18:28:29,344 INFO MainThread:2213050 [wandb_init.py:init():569] calling init triggers +2025-01-07 18:28:29,344 INFO MainThread:2213050 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao'} +2025-01-07 18:28:29,345 INFO MainThread:2213050 [wandb_init.py:init():619] starting backend +2025-01-07 18:28:29,345 INFO MainThread:2213050 [wandb_init.py:init():623] setting up manager +2025-01-07 18:28:29,349 INFO MainThread:2213050 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 18:28:29,350 INFO MainThread:2213050 [wandb_init.py:init():631] backend started and connected +2025-01-07 18:28:29,363 INFO MainThread:2213050 [wandb_init.py:init():720] updated telemetry +2025-01-07 18:28:29,369 INFO MainThread:2213050 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 18:28:29,637 INFO MainThread:2213050 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 18:28:29,703 INFO MainThread:2213050 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 18:28:29,703 INFO MainThread:2213050 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 18:28:30,019 INFO MainThread:2213050 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 18:28:30,019 INFO MainThread:2213050 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 18:28:30,019 INFO MainThread:2213050 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 18:28:30,019 INFO MainThread:2213050 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 18:28:30,021 INFO MainThread:2213050 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 18:28:30,021 INFO MainThread:2213050 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/tb, True +2025-01-07 18:28:38,362 INFO MainThread:2213050 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-07 18:33:13,382 WARNING MsgRouterThr:2213050 [router.py:message_loop():77] message_loop has been closed diff --git a/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/code/FISH/eval_robot.py b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/config.yaml b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23e0d5a2895595872ddbffb640aa7ad269d63954 --- /dev/null +++ b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1017_0' +wandb_notes: + desc: null + value: 1017_0_restarted_11 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736292509 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table/eval/total_success_rate_ci_6_820e6488246e96d3e291.table.json b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table/eval/total_success_rate_ci_6_820e6488246e96d3e291.table.json new file mode 100644 index 0000000000000000000000000000000000000000..32f1d2884e71ace5ffcf1bd17eaf676e3bcdd00d --- /dev/null +++ b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table/eval/total_success_rate_ci_6_820e6488246e96d3e291.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[33, 45, 0.7333333333333333, 0.5597538351893128, 0.8654126673206018, 0.5956548750209234, 0.8532125278600939, 0.6185450404645343, 0.8272674604595824, 0.6349873728780184, 0.8171870484313783]]} \ No newline at end of file diff --git a/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/0_eval_0_7ad56c694ef7147e5404.mp4 b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/0_eval_0_7ad56c694ef7147e5404.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1d1b66271f2a40dc6dcbde6afed22a8155571baa --- /dev/null +++ b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/0_eval_0_7ad56c694ef7147e5404.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ad56c694ef7147e5404200e1d7187387e234f4fbe226b1ead59c0c7110db96d +size 1451975 diff --git a/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/1_eval_1_c12723790e5302e09cec.mp4 b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/1_eval_1_c12723790e5302e09cec.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..78a1516ace90208a1b4d0dff9aa4f41cc5e36efe --- /dev/null +++ b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/1_eval_1_c12723790e5302e09cec.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c12723790e5302e09cecfad97ad8bed87763a0f20764126ee63807b24aabed94 +size 1430855 diff --git a/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/2_eval_2_85ba534ec05635d71996.mp4 b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/2_eval_2_85ba534ec05635d71996.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68fd3f0d5250c6a457a86ff271b98c193b321ae9 --- /dev/null +++ b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/2_eval_2_85ba534ec05635d71996.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85ba534ec05635d7199667d49fd66950ba507f4b70eaa3430428a01b705eeee2 +size 1433745 diff --git a/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/3_eval_3_88082439ff7f6a045e2d.mp4 b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/3_eval_3_88082439ff7f6a045e2d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2028206ca374a35c8ecb333ab819c67efd433e63 --- /dev/null +++ b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/3_eval_3_88082439ff7f6a045e2d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88082439ff7f6a045e2d1cbfbebbf279576bc2d88a9501160a136b0d0ee7835b +size 1434700 diff --git a/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/4_eval_4_df2f032446cb0a313989.mp4 b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/4_eval_4_df2f032446cb0a313989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..090b6ffd3506210a33ae4853670ca76579a47d48 --- /dev/null +++ b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/4_eval_4_df2f032446cb0a313989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df2f032446cb0a313989504722ba4794f92400a01508c67b78cf85f246b1301a +size 1432052 diff --git a/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c04265ae9fb2d034704f56ee5593ef83f4019f82 --- /dev/null +++ b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log @@ -0,0 +1,277 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736292518.363419]: resetting environment +[INFO] [1736292518.370966]: cleared current plan +[INFO] [1736292518.371763]: moving to home +[INFO] [1736292519.373202]: reached home +[INFO] [1736292519.374525]: reset action history +[INFO] [1736292522.084146]: environment reset +Starting episode 0 +[INFO] [1736292522.093754]: resetting environment +[INFO] [1736292522.101395]: cleared current plan +[INFO] [1736292522.102674]: moving to home +[INFO] [1736292523.104776]: reached home +[INFO] [1736292523.106272]: reset action history +[INFO] [1736292525.815239]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736292529.834657]: resetting environment +[INFO] [1736292529.839229]: cleared current plan +[INFO] [1736292529.840055]: moving to home +[INFO] [1736292530.841259]: reached home +[INFO] [1736292530.842615]: reset action history +[INFO] [1736292533.550729]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/episode_rosbags/episode_0_2025-01-07-18-28-53.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2213050.log +started roslaunch server http://158.130.50.37:43915/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2213343] +process[print_text-2]: started with pid [2213367] +process[rosbag_record-3]: started with pid [2213368] +started bagging! +For topic gripper_width: timestamp difference is -116222875 for nearest: 1736292535478798373 - target: 1736292535595021248 at idx 26 +For topic gripper_width: timestamp difference is -111372490 for nearest: 1736292541478801470 - target: 1736292541590173960 at idx 93 +[INFO] [1736292568.703431]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736292569.429771]: Stored episode 1. +[INFO] [1736292569.430038]: Saving video... +[INFO] [1736292569.628921]: Video saved! +Starting episode 1 +[INFO] [1736292569.642539]: resetting environment +[INFO] [1736292569.654394]: cleared current plan +[INFO] [1736292569.656522]: moving to home +[INFO] [1736292573.359411]: reached home +[INFO] [1736292573.361308]: reset action history +[INFO] [1736292576.071197]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736292577.358617]: resetting environment +[INFO] [1736292577.362221]: cleared current plan +[INFO] [1736292577.362430]: moving to home +[INFO] [1736292578.363971]: reached home +[INFO] [1736292578.364384]: reset action history +[INFO] [1736292581.071041]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/episode_rosbags/episode_1_2025-01-07-18-29-41.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2213050.log +started roslaunch server http://158.130.50.37:46467/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2213503] +process[print_text-5]: started with pid [2213504] +process[rosbag_record-6]: started with pid [2213528] +started bagging! +For topic gripper_width: timestamp difference is -106321252 for nearest: 1736292589178782307 - target: 1736292589285103559 at idx 111 +[INFO] [1736292615.851995]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1736292616.638320]: Stored episode 2. +[INFO] [1736292616.638672]: Saving video... +[INFO] [1736292617.328518]: Video saved! +Starting episode 2 +[INFO] [1736292617.355453]: resetting environment +[INFO] [1736292617.371472]: cleared current plan +[INFO] [1736292617.371706]: moving to home +[WARN] [1736292617.191220]: Plan exhausted +[WARN] [1736292617.231637]: Plan exhausted +[WARN] [1736292617.271301]: Plan exhausted +[WARN] [1736292617.310333]: Plan exhausted +[WARN] [1736292617.350357]: Plan exhausted +[INFO] [1736292621.075642]: reached home +[INFO] [1736292621.085693]: reset action history +[INFO] [1736292623.874083]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736292627.146491]: resetting environment +[INFO] [1736292627.158270]: cleared current plan +[INFO] [1736292627.158498]: moving to home +[INFO] [1736292628.160821]: reached home +[INFO] [1736292628.161698]: reset action history +[INFO] [1736292630.871353]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/episode_rosbags/episode_2_2025-01-07-18-30-30.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2213050.log +started roslaunch server http://158.130.50.37:34665/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2213659] +process[print_text-8]: started with pid [2213660] +process[rosbag_record-9]: started with pid [2213661] +started bagging! +For topic gripper_width: timestamp difference is 101674715 for nearest: 1736292640445465485 - target: 1736292640343790770 at idx 132 +For topic gripper_width: timestamp difference is 109650000 for nearest: 1736292650345464810 - target: 1736292650235814810 at idx 249 +[INFO] [1736292665.680992]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736292666.439954]: Stored episode 3. +[INFO] [1736292666.442764]: Saving video... +[INFO] [1736292666.782830]: Video saved! +Starting episode 3 +[INFO] [1736292666.837006]: resetting environment +[INFO] [1736292666.864243]: cleared current plan +[INFO] [1736292666.864472]: moving to home +[INFO] [1736292670.568181]: reached home +[INFO] [1736292670.571230]: reset action history +[INFO] [1736292673.349248]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736292683.268902]: resetting environment +[INFO] [1736292683.290830]: cleared current plan +[INFO] [1736292683.291095]: moving to home +[INFO] [1736292684.293533]: reached home +[INFO] [1736292684.300134]: reset action history +[INFO] [1736292687.084555]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/episode_rosbags/episode_3_2025-01-07-18-31-27.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2213050.log +started roslaunch server http://158.130.50.37:37099/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2213813] +process[print_text-11]: started with pid [2213814] +process[rosbag_record-12]: started with pid [2213838] +started bagging! +[INFO] [1736292721.984129]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit[pub_text-10] killing on exit +[INFO] [1736292722.771351]: Stored episode 4. +[INFO] [1736292722.794189]: Saving video... +[INFO] [1736292723.128844]: Video saved! +Starting episode 4 +[INFO] [1736292723.196888]: resetting environment +[INFO] [1736292723.235595]: cleared current plan +[INFO] [1736292723.236153]: moving to home +[INFO] [1736292726.944342]: reached home +[INFO] [1736292726.951047]: reset action history +[INFO] [1736292729.755227]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736292730.793090]: resetting environment +[INFO] [1736292730.825681]: cleared current plan +[INFO] [1736292730.825932]: moving to home +[INFO] [1736292731.829202]: reached home +[INFO] [1736292731.830993]: reset action history +[INFO] [1736292734.623419]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/episode_rosbags/episode_4_2025-01-07-18-32-14.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2213050.log +started roslaunch server http://158.130.50.37:42153/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2213955] +process[print_text-14]: started with pid [2213956] +process[rosbag_record-15]: started with pid [2213980] +started bagging! +[INFO] [1736292769.569174]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736292770.327806]: Stored episode 5. +[INFO] [1736292770.332230]: Saving video... +[INFO] [1736292771.052507]: Video saved! +[INFO] [1736292771.098588]: resetting environment +[INFO] [1736292771.149913]: cleared current plan +[INFO] [1736292771.151833]: moving to home +[WARN] [1736292770.870769]: Plan exhausted +[WARN] [1736292770.911993]: Plan exhausted +[WARN] [1736292770.957263]: Plan exhausted +[WARN] [1736292770.993368]: Plan exhausted +[WARN] [1736292771.032091]: Plan exhausted +[WARN] [1736292771.071576]: Plan exhausted +[WARN] [1736292771.113288]: Plan exhausted +[INFO] [1736292774.859751]: reached home +[INFO] [1736292774.867609]: reset action history +[INFO] [1736292777.670869]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/requirements.txt b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-metadata.json b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..2f96cd5727106aa1e85ee883b0cbd9a6f36cd53c --- /dev/null +++ b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T23:28:29.728596", + "startedAt": "2025-01-07T23:28:29.340605", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3561.709375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.97, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4496.86, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 850.1733322143555 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..185db11ba47e2e3e6100ac3abe32318ae5e2c599 --- /dev/null +++ b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "7ad56c694ef7147e5404200e1d7187387e234f4fbe226b1ead59c0c7110db96d", "size": 1451975, "path": "media/videos/eval/0_eval_0_7ad56c694ef7147e5404.mp4"}, "global_step": 665, "_timestamp": 1736292780.1484807, "_runtime": 270.7966537475586, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "c12723790e5302e09cecfad97ad8bed87763a0f20764126ee63807b24aabed94", "size": 1430855, "path": "media/videos/eval/1_eval_1_c12723790e5302e09cec.mp4"}, "eval/num_success": 5.0, "episode": 45.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "85ba534ec05635d7199667d49fd66950ba507f4b70eaa3430428a01b705eeee2", "size": 1433745, "path": "media/videos/eval/2_eval_2_85ba534ec05635d71996.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "88082439ff7f6a045e2d1cbfbebbf279576bc2d88a9501160a136b0d0ee7835b", "size": 1434700, "path": "media/videos/eval/3_eval_3_88082439ff7f6a045e2d.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "df2f032446cb0a313989504722ba4794f92400a01508c67b78cf85f246b1301a", "size": 1432052, "path": "media/videos/eval/4_eval_4_df2f032446cb0a313989.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://ntw8xuael9ha5kiqq1lznncwo36n47jlgd434ii7da8xv9su3jr4dypigl5dqnnrrfeqbf965hsl87q4pva441tvux0ghr10qb0xmggoovew8zub67ogjxubk7cpkgrg/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://qt7zy8lkn5a1rlwsnl4wpb1se1nvv1xe3dkr2pqomztbqdgbyuivgb36pcdq2ln601rsd00ucqa6v1bt9h8xfu1evrbxro7qu1wbt7pmj85ncsnk4wa7r4o0noi45tme:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 33, "total_num_episodes": 45, "total_success_rate": 0.7333333333333333, "total_success_rate_lb_0.01": 0.5597538351893128, "total_success_rate_ub_0.01": 0.8654126673206018, "total_success_rate_lb_0.025": 0.5956548750209234, "total_success_rate_ub_0.025": 0.8532125278600939, "total_success_rate_lb_0.05": 0.6185450404645343, "total_success_rate_ub_0.05": 0.8272674604595824, "total_success_rate_lb_0.1": 0.6349873728780184, "total_success_rate_ub_0.1": 0.8171870484313783, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "820e6488246e96d3e291b745724701cb6fa813e57b8bf65572c3640a3015ff29", "size": 520, "artifact_path": "wandb-client-artifact://mrjed3hipk0s20ojwh4xve0iic4zhqot539n4e2ubxmj2bjrd81ajrehd6i1kdfssxo3225xpglc3n9f6x7cp8budjqfwbnymoyybbv1otivcoe1c7h366y2c4jpmntn/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://w5ro3ca9803t2594fgon7pk59w19xtnjuoycoxg4cg3v2oqjs65y6klpws68s9ricmk5f12f6xmrhx78wrhuupj1qke1045u62g517q92e4i31ug3g5r304ly2aqpgu3:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_820e6488246e96d3e291.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 272}, "eval/total_success_rate": 0.7333333492279053, "eval/total_success_rate_lb_0.01": 0.5597538352012634, "eval/total_success_rate_ub_0.01": 0.8654126524925232, "eval/total_success_rate_lb_0.025": 0.5956548452377319, "eval/total_success_rate_ub_0.025": 0.8532125353813171, "eval/total_success_rate_lb_0.05": 0.6185450553894043, "eval/total_success_rate_ub_0.05": 0.8272674679756165, "eval/total_success_rate_lb_0.1": 0.6349873542785645, "eval/total_success_rate_ub_0.1": 0.8171870708465576} \ No newline at end of file diff --git a/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/logs/debug-internal.log b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..609942e351027aa009e8dca9dc42d81223633d5c --- /dev/null +++ b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/logs/debug-internal.log @@ -0,0 +1,769 @@ +2025-01-07 18:28:29,351 INFO StreamThr :2213249 [internal.py:wandb_internal():85] W&B internal server running at pid: 2213249, started at: 2025-01-07 18:28:29.350862 +2025-01-07 18:28:29,352 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status +2025-01-07 18:28:29,364 INFO WriterThread:2213249 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/run-x85ya2ao.wandb +2025-01-07 18:28:29,364 DEBUG SenderThread:2213249 [sender.py:send():379] send: header +2025-01-07 18:28:29,371 DEBUG SenderThread:2213249 [sender.py:send():379] send: run +2025-01-07 18:28:29,623 INFO SenderThread:2213249 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files +2025-01-07 18:28:29,623 INFO SenderThread:2213249 [sender.py:_start_run_threads():1188] run started: x85ya2ao with start time 1736292509.351827 +2025-01-07 18:28:29,637 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 18:28:29,637 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: check_version +2025-01-07 18:28:29,710 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 18:28:29,714 DEBUG HandlerThread:2213249 [system_info.py:__init__():26] System info init +2025-01-07 18:28:29,714 DEBUG HandlerThread:2213249 [system_info.py:__init__():41] System info init done +2025-01-07 18:28:29,714 INFO HandlerThread:2213249 [system_monitor.py:start():194] Starting system monitor +2025-01-07 18:28:29,715 INFO SystemMonitor:2213249 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 18:28:29,715 INFO HandlerThread:2213249 [system_monitor.py:probe():214] Collecting system info +2025-01-07 18:28:29,715 INFO SystemMonitor:2213249 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 18:28:29,716 INFO SystemMonitor:2213249 [interfaces.py:start():188] Started disk monitoring +2025-01-07 18:28:29,716 INFO SystemMonitor:2213249 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 18:28:29,717 INFO SystemMonitor:2213249 [interfaces.py:start():188] Started memory monitoring +2025-01-07 18:28:29,717 INFO SystemMonitor:2213249 [interfaces.py:start():188] Started network monitoring +2025-01-07 18:28:29,728 DEBUG HandlerThread:2213249 [system_info.py:probe():152] Probing system +2025-01-07 18:28:29,730 DEBUG HandlerThread:2213249 [system_info.py:_probe_git():137] Probing git +2025-01-07 18:28:29,734 DEBUG HandlerThread:2213249 [system_info.py:_probe_git():145] Probing git done +2025-01-07 18:28:29,734 DEBUG HandlerThread:2213249 [system_info.py:probe():200] Probing system done +2025-01-07 18:28:29,734 DEBUG HandlerThread:2213249 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T23:28:29.728596', 'startedAt': '2025-01-07T23:28:29.340605', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.709375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.97, 'min': 800.0, 'max': 4700.0}, {'current': 4496.86, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 850.1733322143555}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 18:28:29,734 INFO HandlerThread:2213249 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 18:28:29,734 INFO HandlerThread:2213249 [system_monitor.py:probe():227] Publishing system info +2025-01-07 18:28:29,734 DEBUG HandlerThread:2213249 [system_info.py:_save_code():44] Saving code +2025-01-07 18:28:29,736 DEBUG HandlerThread:2213249 [system_info.py:_save_code():67] Saving code done +2025-01-07 18:28:29,736 DEBUG HandlerThread:2213249 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 18:28:29,776 DEBUG HandlerThread:2213249 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 18:28:29,776 INFO HandlerThread:2213249 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 18:28:29,780 DEBUG SenderThread:2213249 [sender.py:send():379] send: files +2025-01-07 18:28:29,780 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 18:28:29,780 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 18:28:30,018 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 18:28:30,019 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: python_packages +2025-01-07 18:28:30,019 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:30,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:28:30,020 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:28:30,023 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/1a96eit6-wandb-metadata.json +2025-01-07 18:28:30,061 INFO HandlerThread:2213249 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/tb" + save: true +} +control { + relay_id: "4cbe04b595d84eb49319b193420ac055" +} +_info { + stream_id: "x85ya2ao" +} + +2025-01-07 18:28:30,061 DEBUG HandlerThread:2213249 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 18:28:30,068 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/qxdyx2pg-code/FISH/eval_robot.py +2025-01-07 18:28:30,152 DEBUG SenderThread:2213249 [sender.py:send():379] send: telemetry +2025-01-07 18:28:30,152 DEBUG SenderThread:2213249 [sender.py:send():379] send: tbrecord +2025-01-07 18:28:30,153 DEBUG SenderThread:2213249 [sender.py:send():379] send: files +2025-01-07 18:28:30,153 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 with policy live +2025-01-07 18:28:30,326 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/kteafxg6-events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:28:30,625 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/requirements.txt +2025-01-07 18:28:30,625 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:28:30,625 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:28:30,625 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/code/FISH/eval_robot.py +2025-01-07 18:28:30,625 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-metadata.json +2025-01-07 18:28:30,625 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/code +2025-01-07 18:28:30,625 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/code/FISH +2025-01-07 18:28:31,019 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:32,019 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:32,624 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:28:33,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:34,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:35,019 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:28:35,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:36,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:37,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:38,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:38,363 DEBUG SenderThread:2213249 [sender.py:send():379] send: config +2025-01-07 18:28:39,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:40,019 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:28:40,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:40,626 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:28:41,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:42,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:43,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:44,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:44,627 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:28:45,019 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:28:45,019 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:28:45,061 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:45,088 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:28:46,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:47,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:48,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:48,628 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:28:49,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:50,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:50,841 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:28:51,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:52,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:52,629 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:28:53,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:54,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:54,630 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:28:55,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:56,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:56,631 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:28:56,721 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:28:57,023 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:58,023 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:59,023 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:00,019 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:29:00,019 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:29:00,065 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:01,024 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:02,024 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:02,632 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:29:02,717 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:03,024 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:03,632 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/config.yaml +2025-01-07 18:29:04,024 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:05,024 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:06,024 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:07,025 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:08,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:08,025 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:09,025 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:10,025 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:11,025 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:12,025 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:13,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:13,025 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:14,025 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:15,019 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:29:15,019 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:29:15,065 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:16,026 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:17,026 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:18,026 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:18,092 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:19,026 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:20,026 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:21,026 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:22,027 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:23,027 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:24,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:24,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:25,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:26,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:27,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:28,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:29,031 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:29,430 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:29,639 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/0_eval_0_7ad56c694ef7147e5404.mp4 +2025-01-07 18:29:29,639 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval +2025-01-07 18:29:29,639 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media +2025-01-07 18:29:29,639 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos +2025-01-07 18:29:29,642 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:29:29,642 DEBUG SenderThread:2213249 [sender.py:send():379] send: files +2025-01-07 18:29:29,643 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_7ad56c694ef7147e5404.mp4 with policy now +2025-01-07 18:29:29,643 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:29:29,643 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:29:29,644 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:29:29,717 DEBUG SystemMonitor:2213249 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 18:29:29,718 DEBUG SenderThread:2213249 [sender.py:send():379] send: stats +2025-01-07 18:29:29,910 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/h1oef31b-media/videos/eval/0_eval_0_7ad56c694ef7147e5404.mp4 +2025-01-07 18:29:30,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:29:30,020 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:29:30,030 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:30,226 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/f1ypq05s-media/videos/eval/0_eval_0_7ad56c694ef7147e5404.mp4 +2025-01-07 18:29:30,639 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:29:30,639 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:29:31,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:32,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:33,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:34,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:34,640 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:29:35,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:35,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:36,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:37,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:37,642 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:29:37,897 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/h6jgbzsy-events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:29:38,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:38,642 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:29:39,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:40,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:40,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:40,643 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:29:41,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:42,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:42,643 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:29:43,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:44,029 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:45,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:29:45,020 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:29:45,061 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:45,112 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:46,030 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:47,031 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:48,031 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:49,032 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:50,031 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:50,646 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:29:51,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:51,032 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:52,033 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:53,032 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:54,032 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:55,032 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:56,033 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:57,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:29:57,033 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:58,033 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:59,033 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:29:59,718 DEBUG SenderThread:2213249 [sender.py:send():379] send: stats +2025-01-07 18:30:00,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:30:00,020 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:30:00,061 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:01,034 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:02,034 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:02,178 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:03,034 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:04,034 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:05,035 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:06,035 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:07,035 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:08,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:08,035 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:09,035 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:10,036 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:11,036 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:12,036 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:13,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:13,036 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:14,036 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:15,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:30:15,020 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:30:15,065 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:16,036 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:17,036 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:17,350 DEBUG SenderThread:2213249 [sender.py:send():379] send: files +2025-01-07 18:30:17,350 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_c12723790e5302e09cec.mp4 with policy now +2025-01-07 18:30:17,351 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:30:17,351 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:30:17,351 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:30:17,355 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:30:17,615 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/ls47xbfn-media/videos/eval/1_eval_1_c12723790e5302e09cec.mp4 +2025-01-07 18:30:17,654 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:30:17,654 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/1_eval_1_c12723790e5302e09cec.mp4 +2025-01-07 18:30:17,655 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval +2025-01-07 18:30:17,941 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/vex9i1rp-media/videos/eval/1_eval_1_c12723790e5302e09cec.mp4 +2025-01-07 18:30:18,038 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:18,372 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:18,655 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:30:19,042 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:19,655 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:30:20,044 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:21,040 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:22,040 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:22,656 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:30:23,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:23,876 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:24,040 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:25,040 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:26,040 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:26,657 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:30:27,040 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:27,489 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:30:27,489 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:30:27,489 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:30:27,658 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:30:27,658 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:30:28,044 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:28,658 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:30:29,042 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:29,205 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:29,719 DEBUG SenderThread:2213249 [sender.py:send():379] send: stats +2025-01-07 18:30:30,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:30:30,020 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:30:30,061 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:30,659 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:30:31,043 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:32,043 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:32,660 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:30:33,043 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:34,043 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:35,023 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:35,043 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:36,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:37,044 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:38,044 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:39,044 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:40,044 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:40,455 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:41,044 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:42,044 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:42,663 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:30:43,044 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:44,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:45,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:30:45,020 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:30:45,061 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:46,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:46,137 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:47,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:48,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:49,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:50,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:51,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:51,389 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:52,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:52,666 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:30:53,045 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:54,046 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:55,046 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:56,046 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:57,024 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:30:57,047 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:58,046 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:59,046 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:30:59,720 DEBUG SenderThread:2213249 [sender.py:send():379] send: stats +2025-01-07 18:31:00,020 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:31:00,021 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:31:00,061 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:01,047 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:02,047 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:02,087 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:03,047 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:04,047 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:05,047 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:06,048 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:06,671 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:31:06,826 DEBUG SenderThread:2213249 [sender.py:send():379] send: files +2025-01-07 18:31:06,826 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_85ba534ec05635d71996.mp4 with policy now +2025-01-07 18:31:06,831 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:31:06,832 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:31:06,832 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:31:06,833 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:31:07,048 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:07,091 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/zd0gdtxj-media/videos/eval/2_eval_2_85ba534ec05635d71996.mp4 +2025-01-07 18:31:07,671 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:31:07,671 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/2_eval_2_85ba534ec05635d71996.mp4 +2025-01-07 18:31:07,672 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval +2025-01-07 18:31:07,865 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:07,967 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/03fwnkzc-media/videos/eval/2_eval_2_85ba534ec05635d71996.mp4 +2025-01-07 18:31:08,049 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:08,672 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:31:09,053 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:10,059 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:11,055 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:12,061 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:12,673 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:31:13,060 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:13,350 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:14,060 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:14,674 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:31:15,025 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:31:15,025 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:31:15,069 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:16,064 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:17,064 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:18,065 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:19,070 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:19,124 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:20,078 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:21,068 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:22,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:23,070 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:23,316 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:31:23,317 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:31:23,318 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:31:23,677 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:31:23,677 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:31:23,948 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/zrriz22a-events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:31:24,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:24,294 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:24,677 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:31:25,075 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:26,078 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:26,678 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:31:27,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:28,072 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:28,678 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:31:29,072 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:29,363 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:29,720 DEBUG SenderThread:2213249 [sender.py:send():379] send: stats +2025-01-07 18:31:30,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:31:30,021 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:31:30,072 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:31,072 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:32,072 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:33,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:34,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:35,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:35,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:36,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:37,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:38,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:39,073 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:40,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:40,075 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:41,075 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:42,076 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:43,075 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:44,076 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:45,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:31:45,021 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:31:45,076 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:45,092 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:46,076 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:47,076 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:48,076 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:49,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:50,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:51,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:51,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:52,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:53,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:54,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:55,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:56,028 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:31:56,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:57,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:58,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:59,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:31:59,720 DEBUG SenderThread:2213249 [sender.py:send():379] send: stats +2025-01-07 18:32:00,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:32:00,021 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:32:00,077 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:01,078 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:01,145 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:02,078 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:03,088 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:03,185 DEBUG SenderThread:2213249 [sender.py:send():379] send: files +2025-01-07 18:32:03,185 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_88082439ff7f6a045e2d.mp4 with policy now +2025-01-07 18:32:03,195 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:32:03,196 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:32:03,196 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:32:03,197 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:32:03,465 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/urcslq2m-media/videos/eval/3_eval_3_88082439ff7f6a045e2d.mp4 +2025-01-07 18:32:03,690 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:32:03,690 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/3_eval_3_88082439ff7f6a045e2d.mp4 +2025-01-07 18:32:03,690 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval +2025-01-07 18:32:03,982 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/3ycc1n9w-media/videos/eval/3_eval_3_88082439ff7f6a045e2d.mp4 +2025-01-07 18:32:04,090 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:04,690 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:05,093 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:06,090 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:06,947 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:07,095 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:08,100 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:08,692 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:09,108 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:10,092 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:10,692 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:11,093 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:11,372 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:32:11,373 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:32:11,373 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:32:11,693 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:32:11,693 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:32:12,097 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:12,693 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:12,877 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:13,098 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:14,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:15,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:32:15,029 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:32:15,102 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:16,102 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:16,695 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:17,102 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:18,102 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:18,127 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:19,103 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:20,103 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:21,103 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:22,103 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:23,103 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:24,033 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:24,103 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:25,103 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:26,103 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:27,103 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:28,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:29,034 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:29,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:29,721 DEBUG SenderThread:2213249 [sender.py:send():379] send: stats +2025-01-07 18:32:30,021 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:32:30,021 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:32:30,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:31,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:32,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:33,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:34,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:34,176 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:35,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:36,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:37,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:38,104 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:39,105 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:40,033 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:40,105 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:41,105 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:42,105 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:43,105 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:44,105 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:45,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:32:45,022 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:32:45,088 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:45,105 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:46,106 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:47,106 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:48,106 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:49,106 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:50,117 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:50,329 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:50,706 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:51,096 DEBUG SenderThread:2213249 [sender.py:send():379] send: files +2025-01-07 18:32:51,096 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_df2f032446cb0a313989.mp4 with policy now +2025-01-07 18:32:51,099 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:32:51,099 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:32:51,099 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:32:51,100 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:32:51,119 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:51,378 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/mbdgva40-media/videos/eval/4_eval_4_df2f032446cb0a313989.mp4 +2025-01-07 18:32:51,706 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:32:51,707 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/4_eval_4_df2f032446cb0a313989.mp4 +2025-01-07 18:32:51,707 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval +2025-01-07 18:32:51,990 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/5q4z2he1-media/videos/eval/4_eval_4_df2f032446cb0a313989.mp4 +2025-01-07 18:32:52,115 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:52,707 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:53,120 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:53,707 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:54,121 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:55,125 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:56,039 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:32:56,121 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:56,708 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:57,124 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:58,129 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:58,709 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:32:59,132 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:32:59,432 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:32:59,433 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:32:59,433 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:32:59,596 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 18:32:59,596 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 18:32:59,709 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:32:59,709 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:32:59,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:32:59,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media +2025-01-07 18:32:59,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table/eval +2025-01-07 18:32:59,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table +2025-01-07 18:32:59,830 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:32:59,941 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/c2edweux-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:32:59,988 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/qhx6jioi-events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:33:00,022 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:33:00,068 INFO wandb-upload_2:2213249 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzr9fqxos +2025-01-07 18:33:00,130 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:00,235 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 18:33:00,244 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:33:00,491 INFO SenderThread:2213249 [sender.py:send_request_log_artifact():1525] logged artifact run-x85ya2ao-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjEyNzIyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTc2MDEzOA==', 'latestArtifact': None}} +2025-01-07 18:33:00,491 DEBUG SenderThread:2213249 [sender.py:send():379] send: files +2025-01-07 18:33:00,492 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-07 18:33:00,492 DEBUG SenderThread:2213249 [sender.py:send():379] send: stats +2025-01-07 18:33:00,492 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:33:00,492 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,492 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,493 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:33:00,541 DEBUG SenderThread:2213249 [sender.py:send():379] send: telemetry +2025-01-07 18:33:00,542 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,542 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,542 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,542 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,542 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,543 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,543 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,543 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,543 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,544 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,544 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,545 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,545 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,545 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,546 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,546 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,546 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,546 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,546 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,547 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,547 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:00,547 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:00,547 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 18:33:00,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:33:00,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:33:00,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:33:00,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table/eval/total_success_rate_ci_6_820e6488246e96d3e291.table.json +2025-01-07 18:33:00,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table/eval +2025-01-07 18:33:00,747 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/6sq8pmjf-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:33:00,897 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/vbpq42kf-media/table/eval/total_success_rate_ci_6_820e6488246e96d3e291.table.json +2025-01-07 18:33:00,923 INFO wandb-upload_2:2213249 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0qw9bmm0 +2025-01-07 18:33:01,130 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:01,338 INFO SenderThread:2213249 [sender.py:send_request_log_artifact():1525] logged artifact run-x85ya2ao-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjEyNzI1OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTc2MDE0Mg==', 'latestArtifact': None}} +2025-01-07 18:33:01,338 DEBUG SenderThread:2213249 [sender.py:send():379] send: files +2025-01-07 18:33:01,338 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_820e6488246e96d3e291.table.json with policy now +2025-01-07 18:33:01,338 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:33:01,339 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:33:01,339 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:01,339 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:01,601 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/mojljgve-media/table/eval/total_success_rate_ci_6_820e6488246e96d3e291.table.json +2025-01-07 18:33:01,710 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:33:01,746 DEBUG SenderThread:2213249 [sender.py:send():379] send: exit +2025-01-07 18:33:01,746 INFO SenderThread:2213249 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 18:33:01,746 INFO SenderThread:2213249 [sender.py:send_exit():588] handling runtime: 272 +2025-01-07 18:33:01,746 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:01,746 INFO SenderThread:2213249 [sender.py:send_exit():594] send defer +2025-01-07 18:33:01,746 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:01,746 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 18:33:01,747 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:01,747 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 18:33:01,747 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 1 +2025-01-07 18:33:01,747 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:01,747 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 18:33:01,747 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:01,747 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 18:33:01,747 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 2 +2025-01-07 18:33:01,747 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:01,747 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 18:33:01,748 INFO HandlerThread:2213249 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 18:33:01,748 DEBUG SystemMonitor:2213249 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 18:33:01,748 DEBUG SystemMonitor:2213249 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 18:33:01,748 INFO HandlerThread:2213249 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 18:33:01,749 INFO HandlerThread:2213249 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 18:33:01,801 INFO HandlerThread:2213249 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 18:33:01,802 INFO HandlerThread:2213249 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 18:33:01,802 INFO HandlerThread:2213249 [interfaces.py:finish():200] Joined network monitor +2025-01-07 18:33:01,802 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:01,802 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 18:33:01,802 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 3 +2025-01-07 18:33:01,802 DEBUG SenderThread:2213249 [sender.py:send():379] send: stats +2025-01-07 18:33:01,803 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:01,803 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 18:33:01,803 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:01,803 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 18:33:01,803 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 4 +2025-01-07 18:33:01,803 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:01,803 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 18:33:02,711 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:33:02,711 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:33:08,436 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:33:08,436 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:08,436 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:33:08,436 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 18:33:08,436 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 18:33:08,436 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 5 +2025-01-07 18:33:08,437 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:33:08,437 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:08,438 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 18:33:08,439 DEBUG SenderThread:2213249 [sender.py:send():379] send: history +2025-01-07 18:33:08,439 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:33:08,439 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:08,440 DEBUG SenderThread:2213249 [sender.py:send():379] send: summary +2025-01-07 18:33:08,440 INFO SenderThread:2213249 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:33:08,440 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:08,440 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 18:33:08,440 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 6 +2025-01-07 18:33:08,440 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:08,440 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 18:33:08,440 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:08,440 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 18:33:08,444 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:33:08,540 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 7 +2025-01-07 18:33:08,540 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:08,540 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 18:33:08,540 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:08,540 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 18:33:08,713 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/config.yaml +2025-01-07 18:33:08,713 INFO Thread-12 :2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:33:08,747 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:33:11,349 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 8 +2025-01-07 18:33:11,349 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:33:11,349 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:11,349 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 18:33:11,350 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:11,350 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 18:33:11,350 INFO SenderThread:2213249 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 18:33:11,352 INFO SenderThread:2213249 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 18:33:11,354 INFO SenderThread:2213249 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 18:33:11,366 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 9 +2025-01-07 18:33:11,366 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:11,366 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 18:33:11,366 DEBUG SenderThread:2213249 [sender.py:send():379] send: artifact +2025-01-07 18:33:11,561 INFO SenderThread:2213249 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTc1Nzg0Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 18:33:11,561 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:11,561 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 18:33:11,561 INFO SenderThread:2213249 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 18:33:11,714 INFO SenderThread:2213249 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:33:11,715 INFO SenderThread:2213249 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files +2025-01-07 18:33:11,715 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/config.yaml config.yaml +2025-01-07 18:33:11,715 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json wandb-summary.json +2025-01-07 18:33:11,715 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/requirements.txt requirements.txt +2025-01-07 18:33:11,715 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-metadata.json wandb-metadata.json +2025-01-07 18:33:11,715 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:33:11,717 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log output.log +2025-01-07 18:33:11,719 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 18:33:11,720 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/3_eval_3_88082439ff7f6a045e2d.mp4 media/videos/eval/3_eval_3_88082439ff7f6a045e2d.mp4 +2025-01-07 18:33:11,720 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/2_eval_2_85ba534ec05635d71996.mp4 media/videos/eval/2_eval_2_85ba534ec05635d71996.mp4 +2025-01-07 18:33:11,720 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/1_eval_1_c12723790e5302e09cec.mp4 media/videos/eval/1_eval_1_c12723790e5302e09cec.mp4 +2025-01-07 18:33:11,720 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/0_eval_0_7ad56c694ef7147e5404.mp4 media/videos/eval/0_eval_0_7ad56c694ef7147e5404.mp4 +2025-01-07 18:33:11,720 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/videos/eval/4_eval_4_df2f032446cb0a313989.mp4 media/videos/eval/4_eval_4_df2f032446cb0a313989.mp4 +2025-01-07 18:33:11,721 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table/eval/total_success_rate_ci_6_820e6488246e96d3e291.table.json media/table/eval/total_success_rate_ci_6_820e6488246e96d3e291.table.json +2025-01-07 18:33:11,721 INFO SenderThread:2213249 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:33:11,721 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 10 +2025-01-07 18:33:11,721 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:11,722 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 18:33:11,724 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:11,724 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 18:33:11,724 INFO SenderThread:2213249 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:33:11,748 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:33:11,748 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:33:11,895 INFO wandb-upload_0:2213249 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/wandb-summary.json +2025-01-07 18:33:11,916 INFO wandb-upload_2:2213249 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/config.yaml +2025-01-07 18:33:11,919 INFO wandb-upload_1:2213249 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/requirements.txt +2025-01-07 18:33:12,010 INFO wandb-upload_4:2213249 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/files/output.log +2025-01-07 18:33:12,031 INFO wandb-upload_3:2213249 [upload_job.py:push():130] Uploaded file /tmp/tmp_wbyplx4wandb/stpjwhdl-events.out.tfevents.1736292510.leonmkim-ROG-Strix-G15CS-G15CS.2213050.0 +2025-01-07 18:33:12,231 INFO Thread-11 (_thread_body):2213249 [sender.py:transition_state():614] send defer: 11 +2025-01-07 18:33:12,231 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:12,231 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 18:33:12,231 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:12,231 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 18:33:12,232 INFO SenderThread:2213249 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 18:33:12,233 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 12 +2025-01-07 18:33:12,233 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:12,233 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 18:33:12,233 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:12,233 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 18:33:12,233 INFO SenderThread:2213249 [file_stream.py:finish():601] file stream finish called +2025-01-07 18:33:12,325 INFO SenderThread:2213249 [file_stream.py:finish():605] file stream finish is done +2025-01-07 18:33:12,325 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 13 +2025-01-07 18:33:12,325 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:12,325 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 18:33:12,325 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:12,326 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 18:33:12,326 INFO SenderThread:2213249 [sender.py:transition_state():614] send defer: 14 +2025-01-07 18:33:12,326 DEBUG SenderThread:2213249 [sender.py:send():379] send: final +2025-01-07 18:33:12,326 DEBUG SenderThread:2213249 [sender.py:send():379] send: footer +2025-01-07 18:33:12,326 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:33:12,326 INFO HandlerThread:2213249 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 18:33:12,326 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: defer +2025-01-07 18:33:12,326 INFO SenderThread:2213249 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 18:33:12,327 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:33:12,327 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:33:12,327 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 18:33:12,327 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:33:12,328 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: server_info +2025-01-07 18:33:12,328 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 18:33:12,329 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 18:33:12,330 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:12,380 DEBUG SenderThread:2213249 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:33:12,380 INFO MainThread:2213249 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 18:33:12,381 INFO MainThread:2213249 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 18:33:12,381 INFO MainThread:2213249 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 18:33:12,381 DEBUG HandlerThread:2213249 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 18:33:12,381 INFO HandlerThread:2213249 [handler.py:finish():882] shutting down handler +2025-01-07 18:33:13,328 INFO WriterThread:2213249 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/run-x85ya2ao.wandb +2025-01-07 18:33:13,380 INFO SenderThread:2213249 [sender.py:finish():1615] shutting down sender +2025-01-07 18:33:13,380 INFO SenderThread:2213249 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:33:13,380 INFO SenderThread:2213249 [file_pusher.py:join():175] waiting for file pusher diff --git a/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/logs/debug.log b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8cd48ed072ffad3061da06728c6842a66ee24bf2 --- /dev/null +++ b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 18:28:29,343 INFO MainThread:2213050 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 18:28:29,343 INFO MainThread:2213050 [wandb_setup.py:_flush():76] Configure stats pid to 2213050 +2025-01-07 18:28:29,343 INFO MainThread:2213050 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 18:28:29,343 INFO MainThread:2213050 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/settings +2025-01-07 18:28:29,344 INFO MainThread:2213050 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 18:28:29,344 INFO MainThread:2213050 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 18:28:29,344 INFO MainThread:2213050 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 18:28:29,344 INFO MainThread:2213050 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 18:28:29,344 INFO MainThread:2213050 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/logs/debug.log +2025-01-07 18:28:29,344 INFO MainThread:2213050 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/logs/debug-internal.log +2025-01-07 18:28:29,344 INFO MainThread:2213050 [wandb_init.py:init():569] calling init triggers +2025-01-07 18:28:29,344 INFO MainThread:2213050 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao'} +2025-01-07 18:28:29,345 INFO MainThread:2213050 [wandb_init.py:init():619] starting backend +2025-01-07 18:28:29,345 INFO MainThread:2213050 [wandb_init.py:init():623] setting up manager +2025-01-07 18:28:29,349 INFO MainThread:2213050 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 18:28:29,350 INFO MainThread:2213050 [wandb_init.py:init():631] backend started and connected +2025-01-07 18:28:29,363 INFO MainThread:2213050 [wandb_init.py:init():720] updated telemetry +2025-01-07 18:28:29,369 INFO MainThread:2213050 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 18:28:29,637 INFO MainThread:2213050 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 18:28:29,703 INFO MainThread:2213050 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 18:28:29,703 INFO MainThread:2213050 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 18:28:30,019 INFO MainThread:2213050 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 18:28:30,019 INFO MainThread:2213050 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 18:28:30,019 INFO MainThread:2213050 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 18:28:30,019 INFO MainThread:2213050 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 18:28:30,021 INFO MainThread:2213050 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 18:28:30,021 INFO MainThread:2213050 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/x85ya2ao/tb, True +2025-01-07 18:28:38,362 INFO MainThread:2213050 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-07 18:33:13,382 WARNING MsgRouterThr:2213050 [router.py:message_loop():77] message_loop has been closed diff --git a/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/run-x85ya2ao.wandb b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/run-x85ya2ao.wandb new file mode 100644 index 0000000000000000000000000000000000000000..136e90729bdfc0fa8910de6fbacf70f2a2fcf373 Binary files /dev/null and b/x85ya2ao/wandb/run-20250107_182829-x85ya2ao/run-x85ya2ao.wandb differ diff --git a/yjywj48j/episode_rosbags/aligned_depth_to_color_K.npy b/yjywj48j/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/yjywj48j/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/yjywj48j/episode_rosbags/cam_tf_world.npy b/yjywj48j/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/yjywj48j/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/yjywj48j/episode_rosbags/depth_K.npy b/yjywj48j/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/yjywj48j/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/yjywj48j/eval_robot.log b/yjywj48j/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..70c74966555b4bd3de367c8e633dd6be67a7f047 --- /dev/null +++ b/yjywj48j/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 19:57:25,710][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 19:57:25,712][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( +